You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/12/13 19:39:11 UTC
svn commit: r1213844 [1/5] - in /lucene/dev/trunk: ./ lucene/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/contrib/misc/src/java/org/apache/lucene/document/
lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/rege...
Author: rmuir
Date: Tue Dec 13 18:39:06 2011
New Revision: 1213844
URL: http://svn.apache.org/viewvc?rev=1213844&view=rev
Log:
LUCENE-3622: separate IndexDocValues interface from implementation (phase 1)
Added:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/DocValuesField.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/document/DocValuesField.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocValue.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/DocValue.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocValues.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/DocValues.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiDocValues.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/MultiDocValues.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TypePromoter.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/TypePromoter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocProducer.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/codecs/PerDocProducer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/values/
- copied from r1213825, lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/codecs/lucene40/values/
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
- copied unchanged from r1213825, lucene/dev/branches/lucene3622/modules/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
Removed:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/DocValues.java
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/MIGRATE.txt
lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java
lucene/dev/trunk/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Field.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BaseMultiReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexableField.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40DocValuesFormat.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40DocValuesProducer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40FieldInfosReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40FieldInfosWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldInfosReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldInfosWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java
lucene/dev/trunk/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/trunk/modules/benchmark/ (props changed)
lucene/dev/trunk/modules/facet/ (props changed)
lucene/dev/trunk/modules/grouping/CHANGES.txt
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/dv/DVAllGroupHeadsCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/dv/DVAllGroupsCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/dv/DVFirstPassGroupingCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/dv/DVSecondPassGroupingCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionAllGroupHeadsCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionAllGroupsCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionFirstPassGroupingCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/function/FunctionSecondPassGroupingCollector.java
lucene/dev/trunk/modules/grouping/src/java/org/apache/lucene/search/grouping/package.html
lucene/dev/trunk/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
lucene/dev/trunk/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupsCollectorTest.java
lucene/dev/trunk/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/docvalues/BoolDocValues.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatDocValues.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/docvalues/StrDocValues.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/docvalues/StringIndexDocValues.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ConstValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DefFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DivFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleConstValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DualFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IfFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/LinearFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/LiteralValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MinFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiBoolFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NumDocsValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NumericIndexDocValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/PowFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ProductFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/RangeMapFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ReciprocalFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ScaleFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleBoolFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/SimpleFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/SumFloatFunction.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/SumTotalTermFreqValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TotalTermFreqValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/VectorValueSource.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/trunk/solr/ (props changed)
lucene/dev/trunk/solr/CHANGES.txt (props changed)
lucene/dev/trunk/solr/build.xml (props changed)
lucene/dev/trunk/solr/common-build.xml (props changed)
lucene/dev/trunk/solr/contrib/ (props changed)
lucene/dev/trunk/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/trunk/solr/core/ (props changed)
lucene/dev/trunk/solr/core/src/java/ (props changed)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/BoolField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/LatLonType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/RandomSortField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrFieldSource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/ValueSourceParser.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/distance/GeohashFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/distance/GeohashHaversineFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/distance/HaversineConstFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/distance/HaversineFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/distance/SquaredEuclideanFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/distance/StringDistanceFunction.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/distance/VectorDistanceFunction.java
lucene/dev/trunk/solr/core/src/test/ (props changed)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/DummyValueSourceParser.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestIndexSearcher.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/function/NvlValueSourceParser.java
lucene/dev/trunk/solr/example/ (props changed)
lucene/dev/trunk/solr/solrj/ (props changed)
lucene/dev/trunk/solr/solrj/src/java/ (props changed)
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/trunk/solr/test-framework/ (props changed)
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Dec 13 18:39:06 2011
@@ -505,23 +505,23 @@ New features
bytes in RAM. (Mike McCandless)
* LUCENE-3108, LUCENE-2935, LUCENE-2168, LUCENE-1231: Changes from
- IndexDocValues (ColumnStrideFields):
+ DocValues (ColumnStrideFields):
- IndexWriter now supports typesafe dense per-document values stored in
- a column like storage. IndexDocValues are stored on a per-document
+ a column like storage. DocValues are stored on a per-document
basis where each documents field can hold exactly one value of a given
- type. IndexDocValues are provided via Fieldable and can be used in
+ type. DocValues are provided via Fieldable and can be used in
conjunction with stored and indexed values.
- - IndexDocValues provides an entirely RAM resident document id to value
+ - DocValues provides an entirely RAM resident document id to value
mapping per field as well as a DocIdSetIterator based disk-resident
sequential access API relying on filesystem-caches.
- Both APIs are exposed via IndexReader and the Codec / Flex API allowing
- expert users to integrate customized IndexDocValues reader and writer
+ expert users to integrate customized DocValues reader and writer
implementations by extending existing Codecs.
- - IndexDocValues provides implementations for primitive datatypes like int,
+ - DocValues provides implementations for primitive datatypes like int,
long, float, double and arrays of byte. Byte based implementations further
provide storage variants like straight or dereferenced stored bytes, fixed
and variable length bytes as well as index time sorted based on
Modified: lucene/dev/trunk/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/MIGRATE.txt?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/MIGRATE.txt (original)
+++ lucene/dev/trunk/lucene/MIGRATE.txt Tue Dec 13 18:39:06 2011
@@ -383,6 +383,8 @@ LUCENE-1458, LUCENE-2111: Flexible Index
- o.a.l.search.function.ValueSource -> o.a.l.queries.function.ValueSource
- o.a.l.search.function.ValueSourceQuery -> o.a.l.queries.function.FunctionQuery
+ DocValues are now named FunctionValues, to not confuse with Lucene's per-document values.
+
* LUCENE-2392: Enable flexible scoring:
The existing "Similarity" api is now TFIDFSimilarity, if you were extending
Modified: lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Dec 13 18:39:06 2011
@@ -41,12 +41,12 @@ import org.apache.lucene.index.FieldInve
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermState;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.Collector;
@@ -1159,7 +1159,7 @@ public class MemoryIndex {
}
@Override
- public PerDocValues perDocValues() throws IOException {
+ public DocValues docValues(String field) throws IOException {
return null;
}
}
Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java Tue Dec 13 18:39:06 2011
@@ -28,8 +28,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
-import org.apache.lucene.index.values.PerDocFieldValues;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValue;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.util.BytesRef;
/** Defers actually loading a field's value until you ask
@@ -157,20 +157,20 @@ public class LazyDocument {
}
@Override
- public PerDocFieldValues docValues() {
+ public DocValue docValue() {
if (num == 0) {
- return getDocument().getField(name).docValues();
+ return getDocument().getField(name).docValue();
} else {
- return getDocument().getFields(name)[num].docValues();
+ return getDocument().getFields(name)[num].docValue();
}
}
@Override
- public ValueType docValuesType() {
+ public DocValues.Type docValueType() {
if (num == 0) {
- return getDocument().getField(name).docValuesType();
+ return getDocument().getField(name).docValueType();
} else {
- return getDocument().getFields(name)[num].docValuesType();
+ return getDocument().getFields(name)[num].docValueType();
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Field.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Field.java Tue Dec 13 18:39:06 2011
@@ -25,10 +25,10 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.values.PerDocFieldValues;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValue;
import org.apache.lucene.util.BytesRef;
/**
@@ -51,7 +51,7 @@ public class Field implements IndexableF
// pre-analyzed tokenStream for indexed fields
protected TokenStream tokenStream;
// length/offset for all primitive types
- protected PerDocFieldValues docValues;
+ protected DocValue docValue;
protected float boost = 1.0f;
@@ -292,17 +292,17 @@ public class Field implements IndexableF
return result.toString();
}
- public void setDocValues(PerDocFieldValues docValues) {
- this.docValues = docValues;
+ public void setDocValue(DocValue docValue) {
+ this.docValue = docValue;
}
@Override
- public PerDocFieldValues docValues() {
+ public DocValue docValue() {
return null;
}
@Override
- public ValueType docValuesType() {
+ public DocValues.Type docValueType() {
return null;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BaseMultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BaseMultiReader.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BaseMultiReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BaseMultiReader.java Tue Dec 13 18:39:06 2011
@@ -22,7 +22,6 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
@@ -156,9 +155,9 @@ abstract class BaseMultiReader<R extends
public ReaderContext getTopReaderContext() {
return topLevelContext;
}
-
+
@Override
- public PerDocValues perDocValues() throws IOException {
- throw new UnsupportedOperationException("please use MultiPerDocValues.getPerDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Fields");
+ public DocValues docValues(String field) throws IOException {
+ throw new UnsupportedOperationException("please use MultiDocValues#getDocValues, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level DocValues");
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java Tue Dec 13 18:39:06 2011
@@ -17,6 +17,18 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.document.FieldType; // for javadocs
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DocValues.SortedSource;
+import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.codecs.Codec;
+
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
@@ -39,9 +51,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.BlockTreeTermsReader;
-import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -640,11 +649,26 @@ public class CheckIndex {
if (infoStream != null) {
infoStream.print(" test: field norms.........");
}
+ FieldInfos infos = reader.fieldInfos();
byte[] b;
for (final String fieldName : fieldNames) {
+ FieldInfo info = infos.fieldInfo(fieldName);
if (reader.hasNorms(fieldName)) {
b = reader.norms(fieldName);
+ if (b.length != reader.maxDoc()) {
+ throw new RuntimeException("norms for field: " + fieldName + " are of the wrong size");
+ }
+ if (!info.isIndexed || info.omitNorms) {
+ throw new RuntimeException("field: " + fieldName + " should omit norms but has them!");
+ }
++status.totFields;
+ } else {
+ if (reader.norms(fieldName) != null) {
+ throw new RuntimeException("field: " + fieldName + " should omit norms but has them!");
+ }
+ if (info.isIndexed && !info.omitNorms) {
+ throw new RuntimeException("field: " + fieldName + " should have norms but omits them!");
+ }
}
}
@@ -1134,39 +1158,92 @@ public class CheckIndex {
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.hasDocValues()) {
status.totalValueFields++;
- final PerDocValues perDocValues = reader.perDocValues();
- final IndexDocValues docValues = perDocValues.docValues(fieldInfo.name);
+ final DocValues docValues = reader.docValues(fieldInfo.name);
if (docValues == null) {
- continue;
+ throw new RuntimeException("field: " + fieldInfo.name + " omits docvalues but should have them!");
+ }
+ DocValues.Type type = docValues.type();
+ if (type != fieldInfo.getDocValuesType()) {
+ throw new RuntimeException("field: " + fieldInfo.name + " has type: " + type + " but fieldInfos says:" + fieldInfo.getDocValuesType());
}
final Source values = docValues.getDirectSource();
final int maxDoc = reader.maxDoc();
+ int size = docValues.getValueSize();
for (int i = 0; i < maxDoc; i++) {
- switch (fieldInfo.docValues) {
+ switch (fieldInfo.getDocValuesType()) {
case BYTES_FIXED_SORTED:
case BYTES_VAR_SORTED:
case BYTES_FIXED_DEREF:
case BYTES_FIXED_STRAIGHT:
case BYTES_VAR_DEREF:
case BYTES_VAR_STRAIGHT:
- values.getBytes(i, new BytesRef());
+ BytesRef bytes = new BytesRef();
+ values.getBytes(i, bytes);
+ if (size != -1 && size != bytes.length) {
+ throw new RuntimeException("field: " + fieldInfo.name + " returned wrongly sized bytes, was: " + bytes.length + " should be: " + size);
+ }
break;
case FLOAT_32:
+ assert size == 4;
+ values.getFloat(i);
+ break;
case FLOAT_64:
+ assert size == 8;
values.getFloat(i);
break;
case VAR_INTS:
+ assert size == -1;
+ values.getInt(i);
+ break;
case FIXED_INTS_16:
+ assert size == 2;
+ values.getInt(i);
+ break;
case FIXED_INTS_32:
+ assert size == 4;
+ values.getInt(i);
+ break;
case FIXED_INTS_64:
+ assert size == 8;
+ values.getInt(i);
+ break;
case FIXED_INTS_8:
+ assert size == 1;
values.getInt(i);
break;
default:
throw new IllegalArgumentException("Field: " + fieldInfo.name
- + " - no such DocValues type: " + fieldInfo.docValues);
+ + " - no such DocValues type: " + fieldInfo.getDocValuesType());
+ }
+ }
+ if (type == DocValues.Type.BYTES_FIXED_SORTED || type == DocValues.Type.BYTES_VAR_SORTED) {
+ // check sorted bytes
+ SortedSource sortedValues = values.asSortedSource();
+ Comparator<BytesRef> comparator = sortedValues.getComparator();
+ int lastOrd = -1;
+ BytesRef lastBytes = new BytesRef();
+ for (int i = 0; i < maxDoc; i++) {
+ int ord = sortedValues.ord(i);
+ if (ord < 0 || ord > maxDoc) {
+ throw new RuntimeException("field: " + fieldInfo.name + " ord is out of bounds: " + ord);
+ }
+ BytesRef bytes = new BytesRef();
+ sortedValues.getByOrd(ord, bytes);
+ if (lastOrd != -1) {
+ int ordComp = Integer.signum(new Integer(ord).compareTo(new Integer(lastOrd)));
+ int bytesComp = Integer.signum(comparator.compare(bytes, lastBytes));
+ if (ordComp != bytesComp) {
+ throw new RuntimeException("field: " + fieldInfo.name + " ord comparison is wrong: " + ordComp + " comparator claims: " + bytesComp);
+ }
+ }
+ lastOrd = ord;
+ lastBytes = bytes;
}
}
+ } else {
+ if (reader.docValues(fieldInfo.name) != null) {
+ throw new RuntimeException("field: " + fieldInfo.name + " has docvalues but should omit them!");
+ }
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Dec 13 18:39:06 2011
@@ -29,8 +29,6 @@ import java.util.concurrent.ConcurrentHa
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.MapBackedSet;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Tue Dec 13 18:39:06 2011
@@ -26,11 +26,11 @@ import java.util.Map;
import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.DocValuesFormat;
import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.DocValuesFormat;
import org.apache.lucene.index.codecs.FieldInfosWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.values.PerDocFieldValues;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IOUtils;
@@ -224,7 +224,7 @@ final class DocFieldProcessor extends Do
// needs to be more "pluggable" such that if I want
// to have a new "thing" my Fields can do, I can
// easily add it
- FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.fieldType(), false, field.docValuesType());
+ FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.fieldType());
fp = new DocFieldProcessorPerField(this, fi);
fp.next = fieldHash[hashPos];
@@ -235,7 +235,7 @@ final class DocFieldProcessor extends Do
rehash();
}
} else {
- fieldInfos.addOrUpdate(fp.fieldInfo.name, field.fieldType(), false, field.docValuesType());
+ fieldInfos.addOrUpdate(fp.fieldInfo.name, field.fieldType());
}
if (thisFieldGen != fp.lastGen) {
@@ -259,9 +259,9 @@ final class DocFieldProcessor extends Do
if (field.fieldType().stored()) {
fieldsWriter.addField(field, fp.fieldInfo);
}
- final PerDocFieldValues docValues = field.docValues();
- if (docValues != null) {
- docValuesConsumer(docState, fp.fieldInfo).add(docState.docID, docValues);
+ final DocValue docValue = field.docValue();
+ if (docValue != null) {
+ docValuesConsumer(field.docValueType(), docState, fp.fieldInfo).add(docState.docID, docValue);
}
}
@@ -310,12 +310,12 @@ final class DocFieldProcessor extends Do
final private Map<String, DocValuesConsumerAndDocID> docValues = new HashMap<String, DocValuesConsumerAndDocID>();
final private Map<Integer, PerDocConsumer> perDocConsumers = new HashMap<Integer, PerDocConsumer>();
- DocValuesConsumer docValuesConsumer(DocState docState, FieldInfo fieldInfo)
+ DocValuesConsumer docValuesConsumer(DocValues.Type valueType, DocState docState, FieldInfo fieldInfo)
throws IOException {
DocValuesConsumerAndDocID docValuesConsumerAndDocID = docValues.get(fieldInfo.name);
if (docValuesConsumerAndDocID != null) {
if (docState.docID == docValuesConsumerAndDocID.docID) {
- throw new IllegalArgumentException("IndexDocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed, per field)");
+ throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed, per field)");
}
assert docValuesConsumerAndDocID.docID < docState.docID;
docValuesConsumerAndDocID.docID = docState.docID;
@@ -329,17 +329,9 @@ final class DocFieldProcessor extends Do
perDocConsumer = dvFormat.docsConsumer(perDocWriteState);
perDocConsumers.put(0, perDocConsumer);
}
- boolean success = false;
- DocValuesConsumer docValuesConsumer = null;
- try {
- docValuesConsumer = perDocConsumer.addValuesField(fieldInfo);
- fieldInfo.commitDocValues();
- success = true;
- } finally {
- if (!success) {
- fieldInfo.revertUncommitted();
- }
- }
+
+ DocValuesConsumer docValuesConsumer = perDocConsumer.addValuesField(valueType, fieldInfo);
+ fieldInfo.setDocValuesType(valueType);
docValuesConsumerAndDocID = new DocValuesConsumerAndDocID(docValuesConsumer);
docValuesConsumerAndDocID.docID = docState.docID;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Tue Dec 13 18:39:06 2011
@@ -248,7 +248,6 @@ public class DocumentsWriterPerThread {
// mark document as deleted
deleteDocID(docState.docID);
numDocsInRAM++;
- fieldInfos.revertUncommitted();
} else {
abort();
}
@@ -312,7 +311,6 @@ public class DocumentsWriterPerThread {
// Incr here because finishDocument will not
// be called (because an exc is being thrown):
numDocsInRAM++;
- fieldInfos.revertUncommitted();
} else {
abort();
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java Tue Dec 13 18:39:06 2011
@@ -1,6 +1,6 @@
package org.apache.lucene.index;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValues;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,7 +25,7 @@ public final class FieldInfo {
public final int number;
public boolean isIndexed;
- ValueType docValues;
+ private DocValues.Type docValues;
// true if term vector for this field should be stored
@@ -57,7 +57,7 @@ public final class FieldInfo {
*/
public FieldInfo(String name, boolean isIndexed, int number, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, ValueType docValues) {
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) {
this.name = name;
this.isIndexed = isIndexed;
this.number = number;
@@ -118,13 +118,14 @@ public final class FieldInfo {
}
assert this.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !this.storePayloads;
}
- void setDocValues(ValueType v) {
+
+ void setDocValuesType(DocValues.Type v) {
if (docValues == null) {
docValues = v;
}
}
- public void resetDocValues(ValueType v) {
+ public void resetDocValuesType(DocValues.Type v) {
if (docValues != null) {
docValues = v;
}
@@ -134,42 +135,13 @@ public final class FieldInfo {
return docValues != null;
}
- public ValueType getDocValues() {
+ public DocValues.Type getDocValuesType() {
return docValues;
}
-
- private boolean vectorsCommitted;
- private boolean docValuesCommitted;
-
- /**
- * Reverts all uncommitted changes on this {@link FieldInfo}
- * @see #commitVectors()
- */
- void revertUncommitted() {
- if (storeTermVector && !vectorsCommitted) {
- storeOffsetWithTermVector = false;
- storePositionWithTermVector = false;
- storeTermVector = false;
- }
-
- if (docValues != null && !docValuesCommitted) {
- docValues = null;
- }
- }
- /**
- * Commits term vector modifications. Changes to term-vectors must be
- * explicitly committed once the necessary files are created. If those changes
- * are not committed subsequent {@link #revertUncommitted()} will reset the
- * all term-vector flags before the next document.
- */
- void commitVectors() {
- assert storeTermVector;
- vectorsCommitted = true;
- }
-
- void commitDocValues() {
- assert hasDocValues();
- docValuesCommitted = true;
+ public void setStoreTermVectors(boolean withPositions, boolean withOffsets) {
+ storeTermVector = true;
+ storePositionWithTermVector |= withPositions;
+ storeOffsetWithTermVector |= withOffsets;
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java Tue Dec 13 18:39:06 2011
@@ -25,7 +25,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValues;
/** Access to the Field Info file that describes document fields and whether or
* not they are indexed. Each segment has a separate Field Info file. Objects
@@ -299,20 +299,30 @@ public final class FieldInfos implements
*/
synchronized public FieldInfo addOrUpdate(String name, boolean isIndexed, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, ValueType docValues) {
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) {
return addOrUpdateInternal(name, -1, isIndexed, storeTermVector, storePositionWithTermVector,
storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues);
}
- synchronized public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType, boolean scorePayloads, ValueType docValues) {
- return addOrUpdateInternal(name, -1, fieldType.indexed(), fieldType.storeTermVectors(),
- fieldType.storeTermVectorPositions(), fieldType.storeTermVectorOffsets(), fieldType.omitNorms(), scorePayloads,
- fieldType.indexOptions(), docValues);
+ // NOTE: this method does not carry over termVector
+ // booleans nor docValuesType; the indexer chain
+ // (TermVectorsConsumerPerField, DocFieldProcessor) must
+ // set these fields when they succeed in consuming
+ // the document:
+ public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
+ // TODO: really, indexer shouldn't even call this
+ // method (it's only called from DocFieldProcessor);
+ // rather, each component in the chain should update
+ // what it "owns". EG fieldType.indexOptions() should
+ // be updated by maybe FreqProxTermsWriterPerField:
+ return addOrUpdateInternal(name, -1, fieldType.indexed(), false, false, false,
+ fieldType.omitNorms(), false,
+ fieldType.indexOptions(), null);
}
synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, ValueType docValues) {
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValues) {
if (globalFieldNumbers == null) {
throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
}
@@ -322,7 +332,7 @@ public final class FieldInfos implements
fi = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues);
} else {
fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions);
- fi.setDocValues(docValues);
+ fi.setDocValuesType(docValues);
}
version++;
return fi;
@@ -333,7 +343,7 @@ public final class FieldInfos implements
return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed, fi.storeTermVector,
fi.storePositionWithTermVector, fi.storeOffsetWithTermVector,
fi.omitNorms, fi.storePayloads,
- fi.indexOptions, fi.docValues);
+ fi.indexOptions, fi.getDocValuesType());
}
/*
@@ -341,7 +351,7 @@ public final class FieldInfos implements
*/
private FieldInfo addInternal(String name, int fieldNumber, boolean isIndexed,
boolean storeTermVector, boolean storePositionWithTermVector,
- boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, ValueType docValuesType) {
+ boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValues.Type docValuesType) {
// don't check modifiable here since we use that to initially build up FIs
if (globalFieldNumbers != null) {
globalFieldNumbers.setIfNotSet(fieldNumber, name);
@@ -429,16 +439,6 @@ public final class FieldInfos implements
return version;
}
- /**
- * Reverts all uncommitted changes
- * @see FieldInfo#revertUncommitted()
- */
- void revertUncommitted() {
- for (FieldInfo fieldInfo : this) {
- fieldInfo.revertUncommitted();
- }
- }
-
final FieldInfos asReadOnly() {
if (isReadOnly()) {
return this;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Dec 13 18:39:06 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -422,9 +421,9 @@ public class FilterIndexReader extends I
}
@Override
- public PerDocValues perDocValues() throws IOException {
+ public DocValues docValues(String field) throws IOException {
ensureOpen();
- return in.perDocValues();
+ return in.docValues(field);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java Tue Dec 13 18:39:06 2011
@@ -28,8 +28,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
-import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.*;
@@ -772,21 +770,6 @@ public abstract class IndexReader implem
* through them yourself. */
public abstract Fields fields() throws IOException;
- /**
- * Returns {@link PerDocValues} for this reader.
- * This method may return null if the reader has no per-document
- * values stored.
- *
- * <p><b>NOTE</b>: if this is a multi reader ({@link
- * #getSequentialSubReaders} is not null) then this
- * method will throw UnsupportedOperationException. If
- * you really need {@link PerDocValues} for such a reader,
- * use {@link MultiPerDocValues#getPerDocs(IndexReader)}. However, for
- * performance reasons, it's best to get all sub-readers
- * using {@link ReaderUtil#gatherSubReaders} and iterate
- * through them yourself. */
- public abstract PerDocValues perDocValues() throws IOException;
-
public final int docFreq(Term term) throws IOException {
return docFreq(term.field(), term.bytes());
}
@@ -1161,14 +1144,20 @@ public abstract class IndexReader implem
throw new UnsupportedOperationException("This reader does not support this method.");
}
- public final IndexDocValues docValues(String field) throws IOException {
- ensureOpen();
- final PerDocValues perDoc = perDocValues();
- if (perDoc == null) {
- return null;
- }
- return perDoc.docValues(field);
- }
+ /**
+ * Returns {@link DocValues} for this field.
+ * This method may return null if the reader has no per-document
+ * values stored.
+ *
+ * <p><b>NOTE</b>: if this is a multi reader ({@link
+ * #getSequentialSubReaders} is not null) then this
+ * method will throw UnsupportedOperationException. If
+ * you really need {@link DocValues} for such a reader,
+ * use {@link MultiDocValues#getDocValues(IndexReader,String)}. However, for
+ * performance reasons, it's best to get all sub-readers
+ * using {@link ReaderUtil#gatherSubReaders} and iterate
+ * through them yourself. */
+ public abstract DocValues docValues(String field) throws IOException;
private volatile Fields fields;
@@ -1184,21 +1173,6 @@ public abstract class IndexReader implem
return fields;
}
- private volatile PerDocValues perDocValues;
-
- /** @lucene.internal */
- void storePerDoc(PerDocValues perDocValues) {
- ensureOpen();
- this.perDocValues = perDocValues;
- }
-
- /** @lucene.internal */
- PerDocValues retrievePerDoc() {
- ensureOpen();
- return perDocValues;
- }
-
-
/**
* A struct like class that represents a hierarchical relationship between
* {@link IndexReader} instances.
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexableField.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexableField.java Tue Dec 13 18:39:06 2011
@@ -23,8 +23,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.NumericField;
-import org.apache.lucene.index.values.PerDocFieldValues;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.util.BytesRef;
// TODO: how to handle versioning here...?
@@ -77,10 +76,10 @@ public interface IndexableField {
public IndexableFieldType fieldType();
/** Non-null if doc values should be indexed */
- public PerDocFieldValues docValues();
+ public DocValue docValue();
- /** DocValues type; only used if docValues is non-null */
- public ValueType docValuesType();
+ /** DocValues type; only used if docValue is non-null */
+ public DocValues.Type docValueType();
/**
* Creates the TokenStream used for indexing this field. If appropriate,
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/MultiReader.java Tue Dec 13 18:39:06 2011
@@ -21,8 +21,6 @@ import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.MapBackedSet;
/** An IndexReader which reads multiple indexes, appending
@@ -187,5 +185,4 @@ public class MultiReader extends BaseMul
sub.removeReaderFinishedListener(listener);
}
}
-
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/ParallelReader.java Tue Dec 13 18:39:06 2011
@@ -17,8 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.MapBackedSet;
@@ -59,7 +57,6 @@ public class ParallelReader extends Inde
private boolean hasDeletions;
private final ParallelFields fields = new ParallelFields();
- private final ParallelPerDocs perDocs = new ParallelPerDocs();
/** Construct a ParallelReader.
* <p>Note that all subreaders are closed if this ParallelReader is closed.</p>
@@ -132,7 +129,6 @@ public class ParallelReader extends Inde
if (fieldToReader.get(field) == null) {
fieldToReader.put(field, reader);
this.fields.addField(field, MultiFields.getFields(reader).terms(field));
- this.perDocs.addField(field, reader);
}
}
@@ -464,41 +460,10 @@ public class ParallelReader extends Inde
}
}
+ // TODO: I suspect this is completely untested!!!!!
@Override
- public PerDocValues perDocValues() throws IOException {
- ensureOpen();
- return perDocs;
- }
-
- // Single instance of this, per ParallelReader instance
- private static final class ParallelPerDocs extends PerDocValues {
- final TreeMap<String,IndexDocValues> fields = new TreeMap<String,IndexDocValues>();
-
- void addField(String field, IndexReader r) throws IOException {
- PerDocValues perDocs = MultiPerDocValues.getPerDocs(r);
- if (perDocs != null) {
- fields.put(field, perDocs.docValues(field));
- }
- }
-
- @Override
- public void close() throws IOException {
- // nothing to do here
- }
-
- @Override
- public IndexDocValues docValues(String field) throws IOException {
- return fields.get(field);
- }
-
- @Override
- public Collection<String> fields() {
- return fields.keySet();
- }
+ public DocValues docValues(String field) throws IOException {
+ IndexReader reader = fieldToReader.get(field);
+ return reader == null ? null : MultiDocValues.getDocValues(reader, field);
}
}
-
-
-
-
-
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Tue Dec 13 18:39:06 2011
@@ -22,10 +22,10 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.NormsReader;
+import org.apache.lucene.index.codecs.PerDocProducer;
import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.StoredFieldsReader;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.TermVectorsReader;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
@@ -48,7 +48,7 @@ final class SegmentCoreReaders {
final FieldInfos fieldInfos;
final FieldsProducer fields;
- final PerDocValues perDocProducer;
+ final PerDocProducer perDocProducer;
final NormsReader norms;
final Directory dir;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Tue Dec 13 18:39:06 2011
@@ -20,10 +20,13 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader.FieldOption;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldInfosWriter;
import org.apache.lucene.index.codecs.FieldsConsumer;
@@ -31,7 +34,6 @@ import org.apache.lucene.index.codecs.No
import org.apache.lucene.index.codecs.StoredFieldsWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.TermVectorsWriter;
-import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
@@ -131,9 +133,7 @@ final class SegmentMerger {
int numMerged = mergeVectors();
assert numMerged == mergeState.mergedDocCount;
}
- // write FIS once merge is done. IDV might change types or drops fields
- FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
- fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
+
return mergeState;
}
@@ -186,15 +186,40 @@ final class SegmentMerger {
}
}
}
+
+ // returns an updated typepromoter (tracking type and size) given a previous one,
+ // and a newly encountered docvalues
+ private TypePromoter mergeDocValuesType(TypePromoter previous, DocValues docValues) {
+ TypePromoter incoming = TypePromoter.create(docValues.type(), docValues.getValueSize());
+ if (previous == null) {
+ previous = TypePromoter.getIdentityPromoter();
+ }
+ TypePromoter promoted = previous.promote(incoming);
+ if (promoted == null) {
+ // type is incompatible: promote to BYTES_VAR_STRAIGHT
+ return TypePromoter.create(DocValues.Type.BYTES_VAR_STRAIGHT, TypePromoter.VAR_TYPE_VALUE_SIZE);
+ } else {
+ return promoted;
+ }
+ }
private void mergeFieldInfos() throws IOException {
+ // mapping from all docvalues fields found to their promoted types
+ // this is because FieldInfos does not store the valueSize
+ Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
+
for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) {
final IndexReader reader = readerAndLiveDocs.reader;
if (reader instanceof SegmentReader) {
SegmentReader segmentReader = (SegmentReader) reader;
FieldInfos readerFieldInfos = segmentReader.fieldInfos();
for (FieldInfo fi : readerFieldInfos) {
- mergeState.fieldInfos.add(fi);
+ FieldInfo merged = mergeState.fieldInfos.add(fi);
+ // update the type promotion mapping for this reader
+ if (fi.hasDocValues()) {
+ TypePromoter previous = docValuesTypes.get(merged);
+ docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
+ }
}
} else {
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
@@ -209,10 +234,33 @@ final class SegmentMerger {
Collection<String> dvNames = reader.getFieldNames(FieldOption.DOC_VALUES);
mergeState.fieldInfos.addOrUpdate(dvNames, false);
for (String dvName : dvNames) {
- mergeState.fieldInfos.fieldInfo(dvName).setDocValues(reader.docValues(dvName).type());
+ FieldInfo merged = mergeState.fieldInfos.fieldInfo(dvName);
+ DocValues docValues = reader.docValues(dvName);
+ merged.setDocValuesType(docValues.type());
+ TypePromoter previous = docValuesTypes.get(merged);
+ docValuesTypes.put(merged, mergeDocValuesType(previous, docValues));
}
}
}
+
+ // update any promoted doc values types:
+ for (Map.Entry<FieldInfo,TypePromoter> e : docValuesTypes.entrySet()) {
+ FieldInfo fi = e.getKey();
+ TypePromoter promoter = e.getValue();
+ if (promoter == null) {
+ fi.resetDocValuesType(null);
+ } else {
+ assert promoter != TypePromoter.getIdentityPromoter();
+ if (fi.getDocValuesType() != promoter.type()) {
+ // reset the type if we got promoted
+ fi.resetDocValuesType(promoter.type());
+ }
+ }
+ }
+
+ // write the merged infos
+ FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
+ fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
}
/**
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java Tue Dec 13 18:39:06 2011
@@ -27,8 +27,8 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.store.Directory;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.codecs.PerDocProducer;
import org.apache.lucene.index.codecs.StoredFieldsReader;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.TermVectorsReader;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
@@ -425,9 +425,13 @@ public final class SegmentReader extends
}
@Override
- public PerDocValues perDocValues() throws IOException {
+ public DocValues docValues(String field) throws IOException {
ensureOpen();
- return core.perDocProducer;
+ final PerDocProducer perDoc = core.perDocProducer;
+ if (perDoc == null) {
+ return null;
+ }
+ return perDoc.docValues(field);
}
/**
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Tue Dec 13 18:39:06 2011
@@ -26,7 +26,6 @@ import org.apache.lucene.util.ReaderUtil
import org.apache.lucene.index.DirectoryReader; // javadoc
import org.apache.lucene.index.MultiReader; // javadoc
-import org.apache.lucene.index.codecs.PerDocValues;
/**
* This class forces a composite reader (eg a {@link
@@ -35,11 +34,12 @@ import org.apache.lucene.index.codecs.Pe
* IndexReader#getSequentialSubReaders}) to emulate an
* atomic reader. This requires implementing the postings
* APIs on-the-fly, using the static methods in {@link
- * MultiFields}, by stepping through the sub-readers to
- * merge fields/terms, appending docs, etc.
+ * MultiFields}, {@link MultiNorms}, {@link MultiDocValues},
+ * by stepping through the sub-readers to merge fields/terms,
+ * appending docs, etc.
*
* <p>If you ever hit an UnsupportedOperationException saying
- * "please use MultiFields.XXX instead", the simple
+ * "please use MultiXXX.YYY instead", the simple
* but non-performant workaround is to wrap your reader
* using this class.</p>
*
@@ -72,9 +72,9 @@ public final class SlowMultiReaderWrappe
}
@Override
- public PerDocValues perDocValues() throws IOException {
+ public DocValues docValues(String field) throws IOException {
ensureOpen();
- return MultiPerDocValues.getPerDocs(in);
+ return MultiDocValues.getDocValues(in, field);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Tue Dec 13 18:39:06 2011
@@ -113,7 +113,6 @@ final class TermVectorsConsumerPerField
// of a given field in the doc. At this point we flush
// our hash into the DocWriter.
- assert fieldInfo.storeTermVector;
assert termsWriter.vectorFieldsInOrder(fieldInfo);
TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
@@ -150,8 +149,9 @@ final class TermVectorsConsumerPerField
}
termsHashPerField.reset();
+
// commit the termVectors once successful success - FI will otherwise reset them
- fieldInfo.commitVectors();
+ fieldInfo.setStoreTermVectors(doVectorPositions, doVectorOffsets);
}
void shrinkHash() {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java Tue Dec 13 18:39:06 2011
@@ -17,77 +17,52 @@ package org.apache.lucene.index.codecs;
* limitations under the License.
*/
import java.io.IOException;
-import java.util.Collection;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.PerDocFieldValues;
-import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.DocValue;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.Counter;
/**
- * Abstract API that consumes {@link PerDocFieldValues}.
+ * Abstract API that consumes {@link DocValue}s.
* {@link DocValuesConsumer} are always associated with a specific field and
* segments. Concrete implementations of this API write the given
- * {@link PerDocFieldValues} into a implementation specific format depending on
+ * {@link DocValue} into a implementation specific format depending on
* the fields meta-data.
*
* @lucene.experimental
*/
public abstract class DocValuesConsumer {
- protected final Counter bytesUsed;
-
- /**
- * Creates a new {@link DocValuesConsumer}.
- *
- * @param bytesUsed
- * bytes-usage tracking reference used by implementation to track
- * internally allocated memory. All tracked bytes must be released
- * once {@link #finish(int)} has been called.
- */
- protected DocValuesConsumer(Counter bytesUsed) {
- this.bytesUsed = bytesUsed == null ? Counter.newCounter() : bytesUsed;
- }
-
/**
- * Adds the given {@link PerDocFieldValues} instance to this
+ * Adds the given {@link DocValue} instance to this
* {@link DocValuesConsumer}
*
* @param docID
* the document ID to add the value for. The docID must always
* increase or be <tt>0</tt> if it is the first call to this method.
- * @param docValues
- * the values to add
+ * @param docValue
+ * the value to add
* @throws IOException
* if an {@link IOException} occurs
*/
- public abstract void add(int docID, PerDocFieldValues docValues)
+ public abstract void add(int docID, DocValue docValue)
throws IOException;
/**
* Called when the consumer of this API is doc with adding
- * {@link PerDocFieldValues} to this {@link DocValuesConsumer}
+ * {@link DocValue} to this {@link DocValuesConsumer}
*
* @param docCount
* the total number of documents in this {@link DocValuesConsumer}.
* Must be greater than or equal the last given docID to
- * {@link #add(int, PerDocFieldValues)}.
+ * {@link #add(int, DocValue)}.
* @throws IOException
*/
public abstract void finish(int docCount) throws IOException;
/**
- * Gathers files associated with this {@link DocValuesConsumer}
- *
- * @param files
- * the of files to add the consumers files to.
- */
- public abstract void files(Collection<String> files) throws IOException;
-
- /**
* Merges the given {@link org.apache.lucene.index.MergeState} into
* this {@link DocValuesConsumer}.
*
@@ -95,18 +70,18 @@ public abstract class DocValuesConsumer
* the state to merge
* @param docValues docValues array containing one instance per reader (
* {@link org.apache.lucene.index.MergeState#readers}) or <code>null</code> if the reader has
- * no {@link IndexDocValues} instance.
+ * no {@link DocValues} instance.
* @throws IOException
* if an {@link IOException} occurs
*/
- public void merge(MergeState mergeState, IndexDocValues[] docValues) throws IOException {
+ public void merge(MergeState mergeState, DocValues[] docValues) throws IOException {
assert mergeState != null;
boolean hasMerged = false;
for(int readerIDX=0;readerIDX<mergeState.readers.size();readerIDX++) {
final org.apache.lucene.index.MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(readerIDX);
if (docValues[readerIDX] != null) {
hasMerged = true;
- merge(new Writer.SingleSubMergeState(docValues[readerIDX], mergeState.docBase[readerIDX], reader.reader.maxDoc(),
+ merge(new SingleSubMergeState(docValues[readerIDX], mergeState.docBase[readerIDX], reader.reader.maxDoc(),
reader.liveDocs));
}
}
@@ -124,6 +99,9 @@ public abstract class DocValuesConsumer
* @throws IOException
* if an {@link IOException} occurs
*/
+ // TODO: can't we have a default implementation here that merges naively with our apis?
+ // this is how stored fields and term vectors work. its a pain to have to impl merging
+ // (should be an optimization to override it)
protected abstract void merge(SingleSubMergeState mergeState) throws IOException;
/**
@@ -137,7 +115,7 @@ public abstract class DocValuesConsumer
* the source reader for this MergeState - merged values should be read from
* this instance
*/
- public final IndexDocValues reader;
+ public final DocValues reader;
/** the absolute docBase for this MergeState within the resulting segment */
public final int docBase;
/** the number of documents in this MergeState */
@@ -145,7 +123,7 @@ public abstract class DocValuesConsumer
/** the not deleted bits for this MergeState */
public final Bits liveDocs;
- public SingleSubMergeState(IndexDocValues reader, int docBase, int docCount, Bits liveDocs) {
+ public SingleSubMergeState(DocValues reader, int docBase, int docCount, Bits liveDocs) {
assert reader != null;
this.reader = reader;
this.docBase = docBase;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java Tue Dec 13 18:39:06 2011
@@ -27,6 +27,6 @@ import org.apache.lucene.store.Directory
public abstract class DocValuesFormat {
public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException;
- public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException;
+ public abstract PerDocProducer docsProducer(SegmentReadState state) throws IOException;
public abstract void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java Tue Dec 13 18:39:06 2011
@@ -26,23 +26,24 @@ import java.util.TreeMap;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.values.Bytes;
-import org.apache.lucene.index.values.Floats;
-import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.Type; // javadocs
+import org.apache.lucene.index.codecs.lucene40.values.Bytes;
+import org.apache.lucene.index.codecs.lucene40.values.Floats;
+import org.apache.lucene.index.codecs.lucene40.values.Ints;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
/**
- * Abstract base class for PerDocValues implementations
+ * Abstract base class for PerDocProducer implementations
* @lucene.experimental
*/
-public abstract class DocValuesReaderBase extends PerDocValues {
+// TODO: this needs to go under lucene40 codec (its specific to its impl)
+public abstract class DocValuesReaderBase extends PerDocProducer {
protected abstract void closeInternal(Collection<? extends Closeable> closeables) throws IOException;
- protected abstract Map<String, IndexDocValues> docValues();
+ protected abstract Map<String, DocValues> docValues();
@Override
public void close() throws IOException {
@@ -50,24 +51,19 @@ public abstract class DocValuesReaderBas
}
@Override
- public IndexDocValues docValues(String field) throws IOException {
+ public DocValues docValues(String field) throws IOException {
return docValues().get(field);
}
-
- @Override
- public Collection<String> fields() {
- return docValues().keySet();
- }
public Comparator<BytesRef> getComparator() throws IOException {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
// Only opens files... doesn't actually load any values
- protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
+ protected TreeMap<String, DocValues> load(FieldInfos fieldInfos,
String segment, int docCount, Directory dir, IOContext context)
throws IOException {
- TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
+ TreeMap<String, DocValues> values = new TreeMap<String, DocValues>();
boolean success = false;
try {
@@ -79,7 +75,7 @@ public abstract class DocValuesReaderBas
final String id = DocValuesWriterBase.docValuesId(segment,
fieldInfo.number);
values.put(field,
- loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context));
+ loadDocValues(docCount, dir, id, fieldInfo.getDocValuesType(), context));
}
}
success = true;
@@ -93,26 +89,26 @@ public abstract class DocValuesReaderBas
}
/**
- * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
- * Codecs that use different implementations for a certain {@link ValueType} can
+ * Loads a {@link DocValues} instance depending on the given {@link Type}.
+ * Codecs that use different implementations for a certain {@link Type} can
* simply override this method and return their custom implementations.
*
* @param docCount
* number of documents in the segment
* @param dir
- * the {@link Directory} to load the {@link IndexDocValues} from
+ * the {@link Directory} to load the {@link DocValues} from
* @param id
* the unique file ID within the segment
* @param type
* the type to load
- * @return a {@link IndexDocValues} instance for the given type
+ * @return a {@link DocValues} instance for the given type
* @throws IOException
* if an {@link IOException} occurs
* @throws IllegalArgumentException
- * if the given {@link ValueType} is not supported
+ * if the given {@link Type} is not supported
*/
- protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
- ValueType type, IOContext context) throws IOException {
+ protected DocValues loadDocValues(int docCount, Directory dir, String id,
+ DocValues.Type type, IOContext context) throws IOException {
switch (type) {
case FIXED_INTS_16:
case FIXED_INTS_32:
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java?rev=1213844&r1=1213843&r2=1213844&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java Tue Dec 13 18:39:06 2011
@@ -22,7 +22,8 @@ import java.util.Comparator;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.PerDocWriteState;
-import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.codecs.lucene40.values.Writer;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
@@ -32,6 +33,7 @@ import org.apache.lucene.util.Counter;
* Abstract base class for PerDocConsumer implementations
* @lucene.experimental
*/
+//TODO: this needs to go under lucene40 codec (its specific to its impl)
public abstract class DocValuesWriterBase extends PerDocConsumer {
protected final String segmentName;
protected final String segmentSuffix;
@@ -52,8 +54,8 @@ public abstract class DocValuesWriterBas
}
@Override
- public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
- return Writer.create(field.getDocValues(),
+ public DocValuesConsumer addValuesField(DocValues.Type valueType, FieldInfo field) throws IOException {
+ return Writer.create(valueType,
docValuesId(segmentName, field.number),
getDirectory(), getComparator(), bytesUsed, context);
}