You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/10/21 09:32:39 UTC
svn commit: r1633296 [1/2] - in /lucene/dev/trunk: lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lu...
Author: mikemccand
Date: Tue Oct 21 07:32:38 2014
New Revision: 1633296
URL: http://svn.apache.org/r1633296
Log:
LUCENE-6013: remove IndexableFieldType.indexed and FieldInfo.indexed (it's redundant with IndexOptions != null)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Document.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DoubleField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FloatField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/IntField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/LongField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/StringField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/TextField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java
lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java
lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java
lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/document/TestLazyDocument.java
lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
lucene/dev/trunk/lucene/sandbox/src/test/org/apache/lucene/codecs/idversion/StringAndPayloadField.java
lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
lucene/dev/trunk/lucene/spatial/src/test/org/apache/lucene/spatial/DistanceStrategyTest.java
lucene/dev/trunk/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/EnumField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/Insanity.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestStressLucene.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/update/TestExceedMaxTermLength.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Oct 21 07:32:38 2014
@@ -164,6 +164,10 @@ API Changes
will be DocValuesType.NUMERIC if the field indexed and does not omit
norms, else null. (Robert Muir, Mike McCandless)
+* LUCENE-6013: Removed indexed boolean from IndexableFieldType and
+ FieldInfo, since it's redundant with IndexOptions != null. (Robert
+ Muir, Mike McCandless)
+
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java Tue Oct 21 07:32:38 2014
@@ -106,7 +106,7 @@ public class Lucene40FieldInfosFormat ex
// Undead norms! Lucene40NormsReader will check this and bring norms back from the dead:
UndeadNormsProducer.setUndead(attributes);
}
- infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
+ infos[i] = new FieldInfo(name, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, oldValuesType.mapping, -1, Collections.unmodifiableMap(attributes));
}
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java Tue Oct 21 07:32:38 2014
@@ -93,7 +93,7 @@ public class Lucene42FieldInfosFormat ex
UndeadNormsProducer.setUndead(attributes);
}
- infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
+ infos[i] = new FieldInfo(name, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, -1, Collections.unmodifiableMap(attributes));
}
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java Tue Oct 21 07:32:38 2014
@@ -95,7 +95,7 @@ public final class Lucene46FieldInfosFor
UndeadNormsProducer.setUndead(attributes);
}
- infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
+ infos[i] = new FieldInfo(name, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
}
Modified: lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java (original)
+++ lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DocMaker.java Tue Oct 21 07:32:38 2014
@@ -231,7 +231,6 @@ public class DocMaker implements Closeab
// Set ID_FIELD
FieldType ft = new FieldType(valType);
- ft.setIndexed(true);
ft.setStored(true);
Field idField = ds.getField(ID_FIELD, ft);
Modified: lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java (original)
+++ lucene/dev/trunk/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java Tue Oct 21 07:32:38 2014
@@ -72,7 +72,7 @@ public class ReadTokensTask extends Perf
List<Field> fields = doc.getFields();
Analyzer analyzer = getRunData().getAnalyzer();
int tokenCount = 0;
- for(final IndexableField field : fields) {
+ for(final Field field : fields) {
if (!field.fieldType().tokenized() ||
field instanceof IntField ||
field instanceof LongField ||
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java Tue Oct 21 07:32:38 2014
@@ -53,7 +53,6 @@ public class SimpleTextFieldInfosFormat
static final BytesRef NUMFIELDS = new BytesRef("number of fields ");
static final BytesRef NAME = new BytesRef(" name ");
static final BytesRef NUMBER = new BytesRef(" number ");
- static final BytesRef ISINDEXED = new BytesRef(" indexed ");
static final BytesRef STORETV = new BytesRef(" term vectors ");
static final BytesRef STORETVPOS = new BytesRef(" term vector positions ");
static final BytesRef STORETVOFF = new BytesRef(" term vector offsets ");
@@ -89,19 +88,16 @@ public class SimpleTextFieldInfosFormat
assert StringHelper.startsWith(scratch.get(), NUMBER);
int fieldNumber = Integer.parseInt(readString(NUMBER.length, scratch));
- SimpleTextUtil.readLine(input, scratch);
- assert StringHelper.startsWith(scratch.get(), ISINDEXED);
- boolean isIndexed = Boolean.parseBoolean(readString(ISINDEXED.length, scratch));
-
final IndexOptions indexOptions;
- if (isIndexed) {
- SimpleTextUtil.readLine(input, scratch);
- assert StringHelper.startsWith(scratch.get(), INDEXOPTIONS);
- indexOptions = IndexOptions.valueOf(readString(INDEXOPTIONS.length, scratch));
- } else {
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), INDEXOPTIONS);
+ String s = readString(INDEXOPTIONS.length, scratch);
+ if ("null".equals(s)) {
indexOptions = null;
+ } else {
+ indexOptions = IndexOptions.valueOf(s);
}
-
+
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), STORETV);
boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch));
@@ -139,7 +135,7 @@ public class SimpleTextFieldInfosFormat
atts.put(key, value);
}
- infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
+ infos[i] = new FieldInfo(name, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts));
}
@@ -189,16 +185,15 @@ public class SimpleTextFieldInfosFormat
SimpleTextUtil.write(out, Integer.toString(fi.number), scratch);
SimpleTextUtil.writeNewline(out);
- SimpleTextUtil.write(out, ISINDEXED);
- SimpleTextUtil.write(out, Boolean.toString(fi.isIndexed()), scratch);
- SimpleTextUtil.writeNewline(out);
-
- if (fi.isIndexed()) {
+ SimpleTextUtil.write(out, INDEXOPTIONS);
+ IndexOptions indexOptions = fi.getIndexOptions();
+ if (indexOptions != null) {
assert fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads();
- SimpleTextUtil.write(out, INDEXOPTIONS);
SimpleTextUtil.write(out, fi.getIndexOptions().toString(), scratch);
- SimpleTextUtil.writeNewline(out);
+ } else {
+ SimpleTextUtil.write(out, "null", scratch);
}
+ SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, STORETV);
SimpleTextUtil.write(out, Boolean.toString(fi.hasVectors()), scratch);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/NumericTokenStream.java Tue Oct 21 07:32:38 2014
@@ -316,6 +316,12 @@ public final class NumericTokenStream ex
public int getPrecisionStep() {
return precisionStep;
}
+
+ @Override
+ public String toString() {
+ // We override default because it can throw cryptic "illegal shift value":
+ return getClass().getSimpleName() + "(precisionStep=" + precisionStep + " valueSize=" + numericAtt.getValueSize() + " shift=" + numericAtt.getShift() + ")";
+ }
// members
private final NumericTermAttribute numericAtt = addAttribute(NumericTermAttribute.class);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java Tue Oct 21 07:32:38 2014
@@ -49,7 +49,7 @@ import org.apache.lucene.store.IndexOutp
* <li>Header --> {@link CodecUtil#checkSegmentHeader SegmentHeader}</li>
* <li>FieldsCount --> {@link DataOutput#writeVInt VInt}</li>
* <li>FieldName --> {@link DataOutput#writeString String}</li>
- * <li>FieldBits, DocValuesBits --> {@link DataOutput#writeByte Byte}</li>
+ * <li>FieldBits, IndexOptions, DocValuesBits --> {@link DataOutput#writeByte Byte}</li>
* <li>FieldNumber --> {@link DataOutput#writeInt VInt}</li>
* <li>Attributes --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>DocValuesGen --> {@link DataOutput#writeLong(long) Int64}</li>
@@ -64,39 +64,39 @@ import org.apache.lucene.store.IndexOutp
* Lucene, the fields are not numbered implicitly by their order in the
* file, instead explicitly.</li>
* <li>FieldBits: a byte containing field options.
- * <ul>
- * <li>The low-order bit is one for indexed fields, and zero for non-indexed
- * fields.</li>
- * <li>The second lowest-order bit is one for fields that have term vectors
- * stored, and zero for fields without term vectors.</li>
- * <li>If the third lowest order-bit is set (0x4), offsets are stored into
- * the postings list in addition to positions.</li>
- * <li>Fourth bit is unused.</li>
- * <li>If the fifth lowest-order bit is set (0x10), norms are omitted for the
- * indexed field.</li>
- * <li>If the sixth lowest-order bit is set (0x20), payloads are stored for the
- * indexed field.</li>
- * <li>If the seventh lowest-order bit is set (0x40), term frequencies and
- * positions omitted for the indexed field.</li>
- * <li>If the eighth lowest-order bit is set (0x80), positions are omitted for the
- * indexed field.</li>
- * </ul>
- * </li>
- * <li>DocValuesBits: a byte containing per-document value types. The type
- * recorded as two four-bit integers, with the high-order bits representing
- * <code>norms</code> options, and the low-order bits representing
- * {@code DocValues} options. Each four-bit integer can be decoded as such:
- * <ul>
- * <li>0: no DocValues for this field.</li>
- * <li>1: NumericDocValues. ({@link DocValuesType#NUMERIC})</li>
- * <li>2: BinaryDocValues. ({@code DocValuesType#BINARY})</li>
- * <li>3: SortedDocValues. ({@code DocValuesType#SORTED})</li>
- * </ul>
- * </li>
- * <li>DocValuesGen is the generation count of the field's DocValues. If this is -1,
- * there are no DocValues updates to that field. Anything above zero means there
- * are updates stored by {@link DocValuesFormat}.</li>
- * <li>Attributes: a key-value map of codec-private attributes.</li>
+ * <ul>
+ * <li>The low order bit (0x1) is one for fields that have term vectors
+ * stored, and zero for fields without term vectors.</li>
+ * <li>If the second lowest order-bit is set (0x2), norms are omitted for the
+ * indexed field.</li>
+ * <li>If the third lowest-order bit is set (0x4), payloads are stored for the
+ * indexed field.</li>
+ * </ul>
+ * </li>
+ * <li>IndexOptions: a byte containing index options.
+ * <ul>
+ * <li>0: not indexed</li>
+ * <li>1: indexed as DOCS_ONLY</li>
+ * <li>2: indexed as DOCS_AND_FREQS</li>
+ * <li>3: indexed as DOCS_AND_FREQS_AND_POSITIONS</li>
+ * <li>4: indexed as DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS</li>
+ * </ul>
+ * </li>
+ * <li>DocValuesBits: a byte containing per-document value types. The type
+ * recorded as two four-bit integers, with the high-order bits representing
+ * <code>norms</code> options, and the low-order bits representing
+ * {@code DocValues} options. Each four-bit integer can be decoded as such:
+ * <ul>
+ * <li>0: no DocValues for this field.</li>
+ * <li>1: NumericDocValues. ({@link DocValuesType#NUMERIC})</li>
+ * <li>2: BinaryDocValues. ({@code DocValuesType#BINARY})</li>
+ * <li>3: SortedDocValues. ({@code DocValuesType#SORTED})</li>
+ * </ul>
+ * </li>
+ * <li>DocValuesGen is the generation count of the field's DocValues. If this is -1,
+ * there are no DocValues updates to that field. Anything above zero means there
+ * are updates stored by {@link DocValuesFormat}.</li>
+ * <li>Attributes: a key-value map of codec-private attributes.</li>
* </ul>
*
* @lucene.experimental
@@ -109,14 +109,14 @@ public final class Lucene50FieldInfosFor
@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
Throwable priorE = null;
FieldInfo infos[] = null;
try {
- CodecUtil.checkSegmentHeader(input, Lucene50FieldInfosFormat.CODEC_NAME,
- Lucene50FieldInfosFormat.FORMAT_START,
- Lucene50FieldInfosFormat.FORMAT_CURRENT,
+ CodecUtil.checkSegmentHeader(input, CODEC_NAME,
+ FORMAT_START,
+ FORMAT_CURRENT,
segmentInfo.getId(), segmentSuffix);
final int size = input.readVInt(); //read in the size
@@ -129,30 +129,18 @@ public final class Lucene50FieldInfosFor
throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);
}
byte bits = input.readByte();
- boolean isIndexed = (bits & Lucene50FieldInfosFormat.IS_INDEXED) != 0;
- boolean storeTermVector = (bits & Lucene50FieldInfosFormat.STORE_TERMVECTOR) != 0;
- boolean omitNorms = (bits & Lucene50FieldInfosFormat.OMIT_NORMS) != 0;
- boolean storePayloads = (bits & Lucene50FieldInfosFormat.STORE_PAYLOADS) != 0;
- final IndexOptions indexOptions;
- if (!isIndexed) {
- indexOptions = null;
- } else if ((bits & Lucene50FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS) != 0) {
- indexOptions = IndexOptions.DOCS_ONLY;
- } else if ((bits & Lucene50FieldInfosFormat.OMIT_POSITIONS) != 0) {
- indexOptions = IndexOptions.DOCS_AND_FREQS;
- } else if ((bits & Lucene50FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS) != 0) {
- indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
- } else {
- indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
- }
+ boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
+ boolean omitNorms = (bits & OMIT_NORMS) != 0;
+ boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
+
+ final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
// DV Types are packed in one byte
- byte val = input.readByte();
- final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
+ final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
final long dvGen = input.readLong();
final Map<String,String> attributes = input.readStringStringMap();
try {
- infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads,
+ infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
infos[i].checkConsistency();
} catch (IllegalStateException e) {
@@ -168,56 +156,119 @@ public final class Lucene50FieldInfosFor
}
}
+ static {
+ // We "mirror" DocValues enum values with the constants below; let's try to ensure if we add a new DocValuesType while this format is
+ // still used for writing, we remember to fix this encoding:
+ assert DocValuesType.values().length == 5;
+ }
+
+ private static byte docValuesByte(DocValuesType type) {
+ if (type == null) {
+ return 0;
+ } else {
+ switch(type) {
+ case NUMERIC:
+ return 1;
+ case BINARY:
+ return 2;
+ case SORTED:
+ return 3;
+ case SORTED_SET:
+ return 4;
+ case SORTED_NUMERIC:
+ return 5;
+ default:
+ // BUG
+ throw new AssertionError("unhandled DocValuesType: " + type);
+ }
+ }
+ }
+
private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
- if (b == 0) {
+ switch(b) {
+ case 0:
return null;
- } else if (b == 1) {
+ case 1:
return DocValuesType.NUMERIC;
- } else if (b == 2) {
+ case 2:
return DocValuesType.BINARY;
- } else if (b == 3) {
+ case 3:
return DocValuesType.SORTED;
- } else if (b == 4) {
+ case 4:
return DocValuesType.SORTED_SET;
- } else if (b == 5) {
+ case 5:
return DocValuesType.SORTED_NUMERIC;
- } else {
+ default:
throw new CorruptIndexException("invalid docvalues byte: " + b, input);
}
}
+ static {
+ // We "mirror" IndexOptions enum values with the constants below; let's try to ensure if we add a new IndexOption while this format is
+ // still used for writing, we remember to fix this encoding:
+ assert IndexOptions.values().length == 4;
+ }
+
+ private static byte indexOptionsByte(IndexOptions indexOptions) {
+ if (indexOptions == null) {
+ return 0;
+ } else {
+ switch (indexOptions) {
+ case DOCS_ONLY:
+ return 1;
+ case DOCS_AND_FREQS:
+ return 2;
+ case DOCS_AND_FREQS_AND_POSITIONS:
+ return 3;
+ case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS:
+ return 4;
+ default:
+ // BUG:
+ throw new AssertionError("unhandled IndexOptions: " + indexOptions);
+ }
+ }
+ }
+
+ private static IndexOptions getIndexOptions(IndexInput input, byte b) throws IOException {
+ switch (b) {
+ case 0:
+ return null;
+ case 1:
+ return IndexOptions.DOCS_ONLY;
+ case 2:
+ return IndexOptions.DOCS_AND_FREQS;
+ case 3:
+ return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ case 4:
+ return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+ default:
+ // BUG
+ throw new CorruptIndexException("invalid IndexOptions byte: " + b, input);
+ }
+ }
+
@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
- final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene50FieldInfosFormat.EXTENSION);
+ final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
- CodecUtil.writeSegmentHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
+ CodecUtil.writeSegmentHeader(output, CODEC_NAME, FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
fi.checkConsistency();
- IndexOptions indexOptions = fi.getIndexOptions();
- byte bits = 0x0;
- if (fi.hasVectors()) bits |= Lucene50FieldInfosFormat.STORE_TERMVECTOR;
- if (fi.omitsNorms()) bits |= Lucene50FieldInfosFormat.OMIT_NORMS;
- if (fi.hasPayloads()) bits |= Lucene50FieldInfosFormat.STORE_PAYLOADS;
- if (fi.isIndexed()) {
- bits |= Lucene50FieldInfosFormat.IS_INDEXED;
- assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads();
- if (indexOptions == IndexOptions.DOCS_ONLY) {
- bits |= Lucene50FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS;
- } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
- bits |= Lucene50FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS;
- } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) {
- bits |= Lucene50FieldInfosFormat.OMIT_POSITIONS;
- }
- }
+
output.writeString(fi.name);
output.writeVInt(fi.number);
+
+ byte bits = 0x0;
+ if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
+ if (fi.omitsNorms()) bits |= OMIT_NORMS;
+ if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
output.writeByte(bits);
+ output.writeByte(indexOptionsByte(fi.getIndexOptions()));
+
// pack the DV type and hasNorms in one byte
- final byte dv = docValuesByte(fi.getDocValuesType());
- assert (dv & (~0xF)) == 0;
- output.writeByte(dv);
+ output.writeByte(docValuesByte(fi.getDocValuesType()));
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}
@@ -225,24 +276,6 @@ public final class Lucene50FieldInfosFor
}
}
- private static byte docValuesByte(DocValuesType type) {
- if (type == null) {
- return 0;
- } else if (type == DocValuesType.NUMERIC) {
- return 1;
- } else if (type == DocValuesType.BINARY) {
- return 2;
- } else if (type == DocValuesType.SORTED) {
- return 3;
- } else if (type == DocValuesType.SORTED_SET) {
- return 4;
- } else if (type == DocValuesType.SORTED_NUMERIC) {
- return 5;
- } else {
- throw new AssertionError();
- }
- }
-
/** Extension of field infos */
static final String EXTENSION = "fnm";
@@ -252,11 +285,7 @@ public final class Lucene50FieldInfosFor
static final int FORMAT_CURRENT = FORMAT_START;
// Field flags
- static final byte IS_INDEXED = 0x1;
- static final byte STORE_TERMVECTOR = 0x2;
- static final byte STORE_OFFSETS_IN_POSTINGS = 0x4;
- static final byte OMIT_NORMS = 0x10;
- static final byte STORE_PAYLOADS = 0x20;
- static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40;
- static final byte OMIT_POSITIONS = -128;
+ static final byte STORE_TERMVECTOR = 0x1;
+ static final byte OMIT_NORMS = 0x2;
+ static final byte STORE_PAYLOADS = 0x4;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Document.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Document.java Tue Oct 21 07:32:38 2014
@@ -319,7 +319,7 @@ public final class Document implements I
return new FilterIterator<IndexableField, Field>(fields.iterator()) {
@Override
protected boolean predicateFunction(Field field) {
- return field.type.indexed();
+ return field.type.indexOptions() != null;
}
};
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DocumentStoredFieldVisitor.java Tue Oct 21 07:32:38 2014
@@ -69,7 +69,6 @@ public class DocumentStoredFieldVisitor
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
final FieldType ft = new FieldType(TextField.TYPE_STORED);
ft.setStoreTermVectors(fieldInfo.hasVectors());
- ft.setIndexed(fieldInfo.isIndexed());
ft.setOmitNorms(fieldInfo.omitsNorms());
ft.setIndexOptions(fieldInfo.getIndexOptions());
doc.add(new StoredField(fieldInfo.name, value, ft));
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DoubleField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DoubleField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/DoubleField.java Tue Oct 21 07:32:38 2014
@@ -119,7 +119,6 @@ public final class DoubleField extends F
*/
public static final FieldType TYPE_NOT_STORED = new FieldType();
static {
- TYPE_NOT_STORED.setIndexed(true);
TYPE_NOT_STORED.setTokenized(true);
TYPE_NOT_STORED.setOmitNorms(true);
TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
@@ -133,7 +132,6 @@ public final class DoubleField extends F
*/
public static final FieldType TYPE_STORED = new FieldType();
static {
- TYPE_STORED.setIndexed(true);
TYPE_STORED.setTokenized(true);
TYPE_STORED.setOmitNorms(true);
TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java Tue Oct 21 07:32:38 2014
@@ -122,7 +122,7 @@ public class Field implements IndexableF
if (type.stored()) {
throw new IllegalArgumentException("fields with a Reader value cannot be stored");
}
- if (type.indexed() && !type.tokenized()) {
+ if (type.indexOptions() != null && !type.tokenized()) {
throw new IllegalArgumentException("non-tokenized fields must use String values");
}
@@ -148,7 +148,7 @@ public class Field implements IndexableF
if (tokenStream == null) {
throw new NullPointerException("tokenStream cannot be null");
}
- if (!type.indexed() || !type.tokenized()) {
+ if (type.indexOptions() == null || !type.tokenized()) {
throw new IllegalArgumentException("TokenStream fields must be indexed and tokenized");
}
if (type.stored()) {
@@ -214,7 +214,7 @@ public class Field implements IndexableF
if (bytes == null) {
throw new IllegalArgumentException("bytes cannot be null");
}
- if (type.indexed()) {
+ if (type.indexOptions() != null) {
throw new IllegalArgumentException("Fields with BytesRef values cannot be indexed");
}
this.fieldsData = bytes;
@@ -241,7 +241,7 @@ public class Field implements IndexableF
if (value == null) {
throw new IllegalArgumentException("value cannot be null");
}
- if (!type.stored() && !type.indexed()) {
+ if (!type.stored() && type.indexOptions() == null) {
throw new IllegalArgumentException("it doesn't make sense to have a field that "
+ "is neither indexed nor stored");
}
@@ -338,7 +338,7 @@ public class Field implements IndexableF
if (!(fieldsData instanceof BytesRef)) {
throw new IllegalArgumentException("cannot change value type from " + fieldsData.getClass().getSimpleName() + " to BytesRef");
}
- if (type.indexed()) {
+ if (type.indexOptions() != null) {
throw new IllegalArgumentException("cannot set a BytesRef value on an indexed field");
}
if (value == null) {
@@ -419,7 +419,7 @@ public class Field implements IndexableF
* values from stringValue() or getBinaryValue()
*/
public void setTokenStream(TokenStream tokenStream) {
- if (!type.indexed() || !type.tokenized()) {
+ if (type.indexOptions() == null || !type.tokenized()) {
throw new IllegalArgumentException("TokenStream fields must be indexed and tokenized");
}
if (type.numericType() != null) {
@@ -452,7 +452,7 @@ public class Field implements IndexableF
*/
public void setBoost(float boost) {
if (boost != 1.0f) {
- if (type.indexed() == false || type.omitNorms()) {
+ if (type.indexOptions() == null || type.omitNorms()) {
throw new IllegalArgumentException("You cannot set an index-time boost on an unindexed field, or one that omits norms");
}
}
@@ -502,7 +502,8 @@ public class Field implements IndexableF
@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) throws IOException {
- if (!fieldType().indexed()) {
+ if (fieldType().indexOptions() == null) {
+ // Not indexed
return null;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java Tue Oct 21 07:32:38 2014
@@ -43,7 +43,6 @@ public class FieldType implements Indexa
DOUBLE
}
- private boolean indexed;
private boolean stored;
private boolean tokenized = true;
private boolean storeTermVectors;
@@ -51,7 +50,7 @@ public class FieldType implements Indexa
private boolean storeTermVectorPositions;
private boolean storeTermVectorPayloads;
private boolean omitNorms;
- private IndexOptions indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ private IndexOptions indexOptions;
private NumericType numericType;
private boolean frozen;
private int numericPrecisionStep = NumericUtils.PRECISION_STEP_DEFAULT;
@@ -61,7 +60,6 @@ public class FieldType implements Indexa
* Create a new mutable FieldType with all of the properties from <code>ref</code>
*/
public FieldType(FieldType ref) {
- this.indexed = ref.indexed();
this.stored = ref.stored();
this.tokenized = ref.tokenized();
this.storeTermVectors = ref.storeTermVectors();
@@ -100,29 +98,6 @@ public class FieldType implements Indexa
* {@inheritDoc}
* <p>
* The default is <code>false</code>.
- * @see #setIndexed(boolean)
- */
- @Override
- public boolean indexed() {
- return this.indexed;
- }
-
- /**
- * Set to <code>true</code> to index (invert) this field.
- * @param value true if this field should be indexed.
- * @throws IllegalStateException if this FieldType is frozen against
- * future modifications.
- * @see #indexed()
- */
- public void setIndexed(boolean value) {
- checkIfFrozen();
- this.indexed = value;
- }
-
- /**
- * {@inheritDoc}
- * <p>
- * The default is <code>false</code>.
* @see #setStored(boolean)
*/
@Override
@@ -148,7 +123,6 @@ public class FieldType implements Indexa
* The default is <code>true</code>.
* @see #setTokenized(boolean)
*/
- @Override
public boolean tokenized() {
return this.tokenized;
}
@@ -367,7 +341,7 @@ public class FieldType implements Indexa
if (stored()) {
result.append("stored");
}
- if (indexed()) {
+ if (indexOptions != null) {
if (result.length() > 0)
result.append(",");
result.append("indexed");
@@ -441,7 +415,6 @@ public class FieldType implements Indexa
int result = 1;
result = prime * result + ((docValueType == null) ? 0 : docValueType.hashCode());
result = prime * result + ((indexOptions == null) ? 0 : indexOptions.hashCode());
- result = prime * result + (indexed ? 1231 : 1237);
result = prime * result + numericPrecisionStep;
result = prime * result + ((numericType == null) ? 0 : numericType.hashCode());
result = prime * result + (omitNorms ? 1231 : 1237);
@@ -462,7 +435,6 @@ public class FieldType implements Indexa
FieldType other = (FieldType) obj;
if (docValueType != other.docValueType) return false;
if (indexOptions != other.indexOptions) return false;
- if (indexed != other.indexed) return false;
if (numericPrecisionStep != other.numericPrecisionStep) return false;
if (numericType != other.numericType) return false;
if (omitNorms != other.omitNorms) return false;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FloatField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FloatField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FloatField.java Tue Oct 21 07:32:38 2014
@@ -119,7 +119,6 @@ public final class FloatField extends Fi
*/
public static final FieldType TYPE_NOT_STORED = new FieldType();
static {
- TYPE_NOT_STORED.setIndexed(true);
TYPE_NOT_STORED.setTokenized(true);
TYPE_NOT_STORED.setOmitNorms(true);
TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
@@ -134,7 +133,6 @@ public final class FloatField extends Fi
*/
public static final FieldType TYPE_STORED = new FieldType();
static {
- TYPE_STORED.setIndexed(true);
TYPE_STORED.setTokenized(true);
TYPE_STORED.setOmitNorms(true);
TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/IntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/IntField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/IntField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/IntField.java Tue Oct 21 07:32:38 2014
@@ -119,7 +119,6 @@ public final class IntField extends Fiel
*/
public static final FieldType TYPE_NOT_STORED = new FieldType();
static {
- TYPE_NOT_STORED.setIndexed(true);
TYPE_NOT_STORED.setTokenized(true);
TYPE_NOT_STORED.setOmitNorms(true);
TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
@@ -134,7 +133,6 @@ public final class IntField extends Fiel
*/
public static final FieldType TYPE_STORED = new FieldType();
static {
- TYPE_STORED.setIndexed(true);
TYPE_STORED.setTokenized(true);
TYPE_STORED.setOmitNorms(true);
TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/LongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/LongField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/LongField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/LongField.java Tue Oct 21 07:32:38 2014
@@ -129,7 +129,6 @@ public final class LongField extends Fie
*/
public static final FieldType TYPE_NOT_STORED = new FieldType();
static {
- TYPE_NOT_STORED.setIndexed(true);
TYPE_NOT_STORED.setTokenized(true);
TYPE_NOT_STORED.setOmitNorms(true);
TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
@@ -143,7 +142,6 @@ public final class LongField extends Fie
*/
public static final FieldType TYPE_STORED = new FieldType();
static {
- TYPE_STORED.setIndexed(true);
TYPE_STORED.setTokenized(true);
TYPE_STORED.setOmitNorms(true);
TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/StringField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/StringField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/StringField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/StringField.java Tue Oct 21 07:32:38 2014
@@ -36,13 +36,11 @@ public final class StringField extends F
public static final FieldType TYPE_STORED = new FieldType();
static {
- TYPE_NOT_STORED.setIndexed(true);
TYPE_NOT_STORED.setOmitNorms(true);
TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
TYPE_NOT_STORED.setTokenized(false);
TYPE_NOT_STORED.freeze();
- TYPE_STORED.setIndexed(true);
TYPE_STORED.setOmitNorms(true);
TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
TYPE_STORED.setStored(true);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/TextField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/TextField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/TextField.java Tue Oct 21 07:32:38 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.document;
import java.io.Reader;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
/** A field that is indexed and tokenized, without term
* vectors. For example this would be used on a 'body'
@@ -34,11 +35,11 @@ public final class TextField extends Fie
public static final FieldType TYPE_STORED = new FieldType();
static {
- TYPE_NOT_STORED.setIndexed(true);
+ TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE_NOT_STORED.setTokenized(true);
TYPE_NOT_STORED.freeze();
- TYPE_STORED.setIndexed(true);
+ TYPE_STORED.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE_STORED.setTokenized(true);
TYPE_STORED.setStored(true);
TYPE_STORED.freeze();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Tue Oct 21 07:32:38 2014
@@ -361,7 +361,7 @@ final class DefaultIndexingChain extends
}
private static void verifyFieldType(String name, IndexableFieldType ft) {
- if (ft.indexed() == false) {
+ if (ft.indexOptions() == null) {
if (ft.storeTermVectors()) {
throw new IllegalArgumentException("cannot store term vectors "
+ "for a field that is not indexed (field=\"" + name + "\")");
@@ -580,7 +580,6 @@ final class DefaultIndexingChain extends
// reset the TokenStream to the first token
stream.reset();
invertState.setAttributeSource(stream);
-
termsHashPerField.start(field, first);
while (stream.incrementToken()) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Tue Oct 21 07:32:38 2014
@@ -34,7 +34,6 @@ public final class FieldInfo {
/** Internal field number */
public final int number;
- private boolean indexed;
private DocValuesType docValueType;
// True if any document indexed term vectors
@@ -124,14 +123,13 @@ public final class FieldInfo {
*
* @lucene.experimental
*/
- public FieldInfo(String name, boolean indexed, int number, boolean storeTermVector, boolean omitNorms,
+ public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms,
boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
long dvGen, Map<String,String> attributes) {
this.name = name;
- this.indexed = indexed;
this.number = number;
this.docValueType = docValues;
- if (indexed) {
+ if (indexOptions != null) {
this.storeTermVector = storeTermVector;
this.storePayloads = storePayloads;
this.omitNorms = omitNorms;
@@ -152,10 +150,7 @@ public final class FieldInfo {
* Always returns true (or throws IllegalStateException)
*/
public boolean checkConsistency() {
- if (indexed) {
- if (indexOptions == null) {
- throw new IllegalStateException("indexed field '" + name + "' must have index options");
- }
+ if (indexOptions != null) {
// Cannot store payloads unless positions are indexed:
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0 && storePayloads) {
throw new IllegalStateException("indexed field '" + name + "' cannot have payloads without positions");
@@ -183,31 +178,33 @@ public final class FieldInfo {
}
void update(IndexableFieldType ft) {
- update(ft.indexed(), false, ft.omitNorms(), false, ft.indexOptions());
+ update(false, ft.omitNorms(), false, ft.indexOptions());
}
// should only be called by FieldInfos#addOrUpdate
- void update(boolean indexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
+ void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
//System.out.println("FI.update field=" + name + " indexed=" + indexed + " omitNorms=" + omitNorms + " this.omitNorms=" + this.omitNorms);
- this.indexed |= indexed; // once indexed, always indexed
- if (indexed) { // if updated field data is not for indexing, leave the updates out
+ if (this.indexOptions != indexOptions) {
+ if (this.indexOptions == null) {
+ this.indexOptions = indexOptions;
+ } else if (indexOptions != null) {
+ // downgrade
+ this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions : indexOptions;
+ }
+ }
+
+ if (this.indexOptions != null) { // if updated field data is not for indexing, leave the updates out
this.storeTermVector |= storeTermVector; // once vector, always vector
this.storePayloads |= storePayloads;
- if (this.omitNorms != omitNorms) {
+
+ // Awkward: only drop norms if incoming update is indexed:
+ if (indexOptions != null && this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
}
- if (this.indexOptions != indexOptions) {
- if (this.indexOptions == null) {
- this.indexOptions = indexOptions;
- } else {
- // downgrade
- this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions : indexOptions;
- }
- if (this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
- // cannot store payloads if we don't store positions:
- this.storePayloads = false;
- }
- }
+ }
+ if (this.indexOptions == null || this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+ // cannot store payloads if we don't store positions:
+ this.storePayloads = false;
}
assert checkConsistency();
}
@@ -259,7 +256,7 @@ public final class FieldInfo {
}
void setStorePayloads() {
- if (indexed && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
+ if (indexOptions != null && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
storePayloads = true;
}
assert checkConsistency();
@@ -276,14 +273,14 @@ public final class FieldInfo {
* Returns true if this field actually has any norms.
*/
public boolean hasNorms() {
- return indexed && omitNorms == false;
+ return isIndexed() && omitNorms == false;
}
/**
- * Returns true if this field is indexed.
+ * Returns true if this field is indexed (has non-null {@link #getIndexOptions}).
*/
public boolean isIndexed() {
- return indexed;
+ return indexOptions != null;
}
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Tue Oct 21 07:32:38 2014
@@ -284,12 +284,12 @@ public class FieldInfos implements Itera
// rather, each component in the chain should update
// what it "owns". EG fieldType.indexOptions() should
// be updated by maybe FreqProxTermsWriterPerField:
- return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
+ return addOrUpdateInternal(name, -1, false,
fieldType.omitNorms(), false,
fieldType.indexOptions(), fieldType.docValueType());
}
- private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
+ private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
FieldInfo fi = fieldInfo(name);
@@ -300,12 +300,12 @@ public class FieldInfos implements Itera
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues);
- fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
+ fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
assert !byName.containsKey(fi.name);
assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
} else {
- fi.update(isIndexed, storeTermVector, omitNorms, storePayloads, indexOptions);
+ fi.update(storeTermVector, omitNorms, storePayloads, indexOptions);
if (docValues != null) {
// only pay the synchronization cost if fi does not already have a DVType
@@ -323,7 +323,7 @@ public class FieldInfos implements Itera
public FieldInfo add(FieldInfo fi) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
- return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
+ return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getDocValuesType());
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableField.java Tue Oct 21 07:32:38 2014
@@ -18,18 +18,14 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.search.similarities.DefaultSimilarity; // javadocs
import org.apache.lucene.search.similarities.Similarity; // javadocs
-import org.apache.lucene.util.BytesRef;
// TODO: how to handle versioning here...?
-// TODO: we need to break out separate StoredField...
-
/** Represents a single field for indexing. IndexWriter
* consumes Iterable<IndexableField> as a document.
*
@@ -68,7 +64,7 @@ public interface IndexableField extends
* the range of that encoding.
* <p>
* It is illegal to return a boost other than 1.0f for a field that is not
- * indexed ({@link IndexableFieldType#indexed()} is false) or omits normalization values
+ * indexed ({@link IndexableFieldType#indexOptions()} is null) or omits normalization values
* ({@link IndexableFieldType#omitNorms()} returns true).
*
* @see Similarity#computeNorm(FieldInvertState)
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Tue Oct 21 07:32:38 2014
@@ -27,9 +27,6 @@ import org.apache.lucene.index.FieldInfo
*/
public interface IndexableFieldType {
- /** True if this field should be indexed (inverted) */
- public boolean indexed();
-
/** True if the field's value should be stored */
public boolean stored();
@@ -37,7 +34,7 @@ public interface IndexableFieldType {
* True if this field's value should be analyzed by the
* {@link Analyzer}.
* <p>
- * This has no effect if {@link #indexed()} returns false.
+ * This has no effect if {@link #indexOptions()} returns null.
*/
// TODO: shouldn't we remove this? Whether/how a field is
// tokenized is an impl detail under Field?
@@ -51,7 +48,7 @@ public interface IndexableFieldType {
* can be accessed in a document-oriented way from
* {@link IndexReader#getTermVector(int,String)}.
* <p>
- * This option is illegal if {@link #indexed()} returns false.
+ * This option is illegal if {@link #indexOptions()} returns null.
*/
public boolean storeTermVectors();
@@ -91,7 +88,8 @@ public interface IndexableFieldType {
public boolean omitNorms();
/** {@link IndexOptions}, describing what should be
- * recorded into the inverted index */
+ * recorded into the inverted index, or null if this field
+ * is not indexed */
public IndexOptions indexOptions();
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Tue Oct 21 07:32:38 2014
@@ -113,7 +113,7 @@ final class TermVectorsConsumerPerField
@Override
boolean start(IndexableField field, boolean first) {
- assert field.fieldType().indexed();
+ assert field.fieldType().indexOptions() != null;
if (first) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java Tue Oct 21 07:32:38 2014
@@ -143,8 +143,10 @@ public final class NumericUtils {
* @param bytes will contain the encoded value
*/
public static void longToPrefixCodedBytes(final long val, final int shift, final BytesRefBuilder bytes) {
- if ((shift & ~0x3f) != 0) // ensure shift is 0..63
- throw new IllegalArgumentException("Illegal shift value, must be 0..63");
+ // ensure shift is 0..63
+ if ((shift & ~0x3f) != 0) {
+ throw new IllegalArgumentException("Illegal shift value, must be 0..63; got shift=" + shift);
+ }
int nChars = (((63-shift)*37)>>8) + 1; // i/7 is the same as (i*37)>>8 for i in 0..63
bytes.setLength(nChars+1); // one extra for the byte that contains the shift info
bytes.grow(BUF_SIZE_LONG);
@@ -169,8 +171,10 @@ public final class NumericUtils {
* @param bytes will contain the encoded value
*/
public static void intToPrefixCodedBytes(final int val, final int shift, final BytesRefBuilder bytes) {
- if ((shift & ~0x1f) != 0) // ensure shift is 0..31
- throw new IllegalArgumentException("Illegal shift value, must be 0..31");
+ // ensure shift is 0..31
+ if ((shift & ~0x1f) != 0) {
+ throw new IllegalArgumentException("Illegal shift value, must be 0..31; got shift=" + shift);
+ }
int nChars = (((31-shift)*37)>>8) + 1; // i/7 is the same as (i*37)>>8 for i in 0..63
bytes.setLength(nChars+1); // one extra for the byte that contains the shift info
bytes.grow(NumericUtils.BUF_SIZE_LONG); // use the max
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Tue Oct 21 07:32:38 2014
@@ -305,7 +305,6 @@ public class TestMockAnalyzer extends Ba
final RandomIndexWriter writer = new RandomIndexWriter(random(), newDirectory());
final Document doc = new Document();
final FieldType ft = new FieldType();
- ft.setIndexed(true);
ft.setIndexOptions(IndexOptions.DOCS_ONLY);
ft.setTokenized(true);
ft.setStoreTermVectors(true);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestDocument.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestDocument.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestDocument.java Tue Oct 21 07:32:38 2014
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.StorableField;
@@ -64,7 +65,7 @@ public class TestDocument extends Lucene
assertTrue(binaryFld.binaryValue() != null);
assertTrue(binaryFld.fieldType().stored());
- assertFalse(binaryFld.fieldType().indexed());
+ assertNull(binaryFld.fieldType().indexOptions());
String binaryTest = doc.getBinaryValue("binary").utf8ToString();
assertTrue(binaryTest.equals(binaryVal));
@@ -263,7 +264,7 @@ public class TestDocument extends Lucene
FieldType stored = new FieldType();
stored.setStored(true);
FieldType indexedNotTokenized = new FieldType();
- indexedNotTokenized.setIndexed(true);
+ indexedNotTokenized.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
indexedNotTokenized.setTokenized(false);
doc.add(new StringField("keyword", "test1", Field.Store.YES));
doc.add(new StringField("keyword", "test2", Field.Store.YES));
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java Tue Oct 21 07:32:38 2014
@@ -43,7 +43,7 @@ public class TestFieldType extends Lucen
assertFalse(ft4.equals(ft));
FieldType ft5 = new FieldType();
- ft5.setIndexed(true);
+ ft5.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
assertFalse(ft5.equals(ft));
FieldType ft6 = new FieldType();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/Test4GBStoredFields.java Tue Oct 21 07:32:38 2014
@@ -59,7 +59,6 @@ public class Test4GBStoredFields extends
final Document doc = new Document();
final FieldType ft = new FieldType();
- ft.setIndexed(false);
ft.setStored(true);
ft.freeze();
final int valueLength = RandomInts.randomIntBetween(random(), 1 << 13, 1 << 20);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Oct 21 07:32:38 2014
@@ -21,7 +21,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -35,6 +34,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.PhraseQuery;
@@ -1143,7 +1143,7 @@ public class TestAddIndexes extends Luce
IndexWriter w = new IndexWriter(toAdd, conf);
Document doc = new Document();
FieldType customType = new FieldType();
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
doc.add(newField("foo", "bar", customType));
w.addDocument(doc);
w.close();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Tue Oct 21 07:32:38 2014
@@ -93,9 +93,6 @@ public class TestCodecs extends LuceneTe
fieldInfo = fieldInfos.addOrUpdate(name, new IndexableFieldType() {
@Override
- public boolean indexed() { return true; }
-
- @Override
public boolean stored() { return false; }
@Override
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java Tue Oct 21 07:32:38 2014
@@ -276,7 +276,7 @@ public class TestConsistentFieldNumbers
for (FieldInfo fi : fis) {
Field expected = getField(Integer.parseInt(fi.name));
- assertEquals(expected.fieldType().indexed(), fi.isIndexed());
+ assertEquals(expected.fieldType().indexOptions(), fi.getIndexOptions());
assertEquals(expected.fieldType().storeTermVectors(), fi.hasVectors());
}
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestExceedMaxTermLength.java Tue Oct 21 07:32:38 2014
@@ -17,17 +17,18 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import java.io.IOException;
-
-import org.junit.Before;
import org.junit.After;
+import org.junit.Before;
/**
* Tests that a useful exception is thrown when attempting to index a term that is
@@ -58,7 +59,7 @@ public class TestExceedMaxTermLength ext
(dir, newIndexWriterConfig(random(), new MockAnalyzer(random())));
try {
final FieldType ft = new FieldType();
- ft.setIndexed(true);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
ft.setStored(random().nextBoolean());
ft.freeze();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Oct 21 07:32:38 2014
@@ -1247,7 +1247,8 @@ public class TestIndexWriter extends Luc
customType.setTokenized(true);
Field f = new Field("binary", b, 10, 17, customType);
- customType.setIndexed(true);
+ // TODO: this is evil, changing the type after creating the field:
+ customType.setIndexOptions(IndexOptions.DOCS_ONLY);
final MockTokenizer doc1field1 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
doc1field1.setReader(new StringReader("doc1field1"));
f.setTokenStream(doc1field1);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Oct 21 07:32:38 2014
@@ -89,7 +89,7 @@ public class TestIndexWriterExceptions e
custom1.setStoreTermVectorOffsets(true);
custom2.setStored(true);
- custom2.setIndexed(true);
+ custom2.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
custom3.setStored(true);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Tue Oct 21 07:32:38 2014
@@ -47,11 +47,6 @@ public class TestIndexableField extends
private final int counter;
private final IndexableFieldType fieldType = new IndexableFieldType() {
@Override
- public boolean indexed() {
- return (counter % 10) != 3;
- }
-
- @Override
public boolean stored() {
return (counter & 1) == 0 || (counter % 10) == 3;
}
@@ -63,7 +58,7 @@ public class TestIndexableField extends
@Override
public boolean storeTermVectors() {
- return indexed() && counter % 2 == 1 && counter % 10 != 9;
+ return indexOptions() != null && counter % 2 == 1 && counter % 10 != 9;
}
@Override
@@ -88,7 +83,7 @@ public class TestIndexableField extends
@Override
public FieldInfo.IndexOptions indexOptions() {
- return FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+ return counter%10 == 3 ? null : FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
}
@Override
@@ -208,7 +203,7 @@ public class TestIndexableField extends
next = new MyField(finalBaseCount + (fieldUpto++-1));
}
- if (next != null && next.fieldType().indexed()) return true;
+ if (next != null && next.fieldType().indexOptions() != null) return true;
else return this.hasNext();
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Tue Oct 21 07:32:38 2014
@@ -362,7 +362,6 @@ public class TestPostingsOffsets extends
if (i < 99 && random().nextInt(2) == 0) {
// stored only
FieldType ft = new FieldType();
- ft.setIndexed(false);
ft.setStored(true);
doc.add(new Field("foo", "boo!", ft));
} else {
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java Tue Oct 21 07:32:38 2014
@@ -176,7 +176,7 @@ public class TestSegmentReader extends L
// test omit norms
for (int i=0; i<DocHelper.fields.length; i++) {
IndexableField f = DocHelper.fields[i];
- if (f.fieldType().indexed()) {
+ if (f.fieldType().indexOptions() != null) {
assertEquals(reader.getNormValues(f.name()) != null, !f.fieldType().omitNorms());
assertEquals(reader.getNormValues(f.name()) != null, !DocHelper.noNorms.containsKey(f.name()));
if (reader.getNormValues(f.name()) == null) {
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java Tue Oct 21 07:32:38 2014
@@ -31,6 +31,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.TermQuery;
@@ -815,12 +816,12 @@ public class TestStressIndexing2 extends
case 0:
customType.setStored(true);
customType.setOmitNorms(true);
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
customType.freeze();
fields.add(newField(fieldName, getString(1), customType));
break;
case 1:
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
customType.setTokenized(true);
customType.freeze();
fields.add(newField(fieldName, getString(0), customType));
@@ -835,7 +836,7 @@ public class TestStressIndexing2 extends
break;
case 3:
customType.setStored(true);
- customType.setIndexed(true);
+ customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
customType.setTokenized(true);
customType.freeze();
fields.add(newField(fieldName, getString(bigFieldSize), customType));
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java Tue Oct 21 07:32:38 2014
@@ -111,7 +111,7 @@ public class TestSimilarityBase extends
for (int i = 0; i < docs.length; i++) {
Document d = new Document();
FieldType ft = new FieldType(TextField.TYPE_STORED);
- ft.setIndexed(false);
+ ft.setIndexOptions(null);
d.add(newField(FIELD_ID, Integer.toString(i), ft));
d.add(newTextField(FIELD_BODY, docs[i], Field.Store.YES));
writer.addDocument(d);
Modified: lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java (original)
+++ lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/FacetField.java Tue Oct 21 07:32:38 2014
@@ -22,6 +22,7 @@ import java.util.Arrays;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
/**
* Add an instance of this to your {@link Document} for every facet label.
@@ -33,7 +34,7 @@ import org.apache.lucene.document.FieldT
public class FacetField extends Field {
static final FieldType TYPE = new FieldType();
static {
- TYPE.setIndexed(true);
+ TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE.freeze();
}
Modified: lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java (original)
+++ lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetField.java Tue Oct 21 07:32:38 2014
@@ -20,6 +20,7 @@ package org.apache.lucene.facet.sortedse
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.facet.FacetField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
/** Add an instance of this to your Document for every facet
* label to be indexed via SortedSetDocValues. */
@@ -28,7 +29,7 @@ public class SortedSetDocValuesFacetFiel
/** Indexed {@link FieldType}. */
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setIndexed(true);
+ TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE.freeze();
}
Modified: lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java (original)
+++ lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/AssociationFacetField.java Tue Oct 21 07:32:38 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.Facets;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.util.BytesRef;
/** Add an instance of this to your {@link Document} to add
@@ -40,7 +41,7 @@ public class AssociationFacetField exten
/** Indexed {@link FieldType}. */
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setIndexed(true);
+ TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
TYPE.freeze();
}
Modified: lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1633296&r1=1633295&r2=1633296&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Oct 21 07:32:38 2014
@@ -445,7 +445,7 @@ public class MemoryIndex {
if (!fieldInfos.containsKey(fieldName)) {
fieldInfos.put(fieldName,
- new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, -1, null));
+ new FieldInfo(fieldName, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, null, -1, null));
}
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);