You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/08/21 00:47:32 UTC
svn commit: r1516012 [1/4] - in /lucene/dev/branches/branch_4x: ./
dev-tools/ lucene/ lucene/codecs/
lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/
lucene/codecs/src/java/org/apache/lucene/codecs/memory/
lucene/codecs/src/java/org/apache/lucen...
Author: rmuir
Date: Tue Aug 20 22:47:30 2013
New Revision: 1516012
URL: http://svn.apache.org/r1516012
Log:
LUCENE-5178: add missing support for docvalues
Added:
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskNormsFormat.java
- copied unchanged from r1515977, lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskNormsFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
- copied, changed from r1515977, lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
- copied unchanged from r1515977, lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
- copied unchanged from r1515977, lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryDocValuesFormat.java
- copied unchanged from r1515977, lucene/dev/trunk/lucene/codecs/src/test/org/apache/lucene/codecs/memory/TestMemoryDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsConsumer.java
- copied, changed from r1515977, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsConsumer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/
- copied from r1515977, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene45/
- copied from r1515977, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene45/
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet45/
- copied from r1515977, lucene/dev/trunk/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet45/
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/MissingOrdRemapper.java
- copied unchanged from r1515977, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/MissingOrdRemapper.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/
- copied from r1515977, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/package.html
- copied unchanged from r1515977, lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/package.html
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMissing.xml
- copied unchanged from r1515977, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMissing.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/DocValuesMissingTest.java
- copied, changed from r1515977, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesMissingTest.java
Removed:
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42Codec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardNormsFormat.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/package.html
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/package.html
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/package.html
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestNamedSPILoader.java
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/codecs/facet42/Facet42DocValuesProducer.java
lucene/dev/branches/branch_4x/lucene/facet/src/java/org/apache/lucene/facet/util/FacetsPayloadMigrationReader.java
lucene/dev/branches/branch_4x/lucene/facet/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/FacetTestCase.java
lucene/dev/branches/branch_4x/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/DoubleFieldSource.java
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
lucene/dev/branches/branch_4x/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/xsl/index.xsl
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingCodec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardCodec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastCompressingCodec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/FastDecompressionCompressingCodec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/FieldFilterAtomicReader.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
lucene/dev/branches/branch_4x/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/NumericFacets.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/SchemaField.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestDocSet.java
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/example/solr/collection1/conf/schema.xml
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue Aug 20 22:47:30 2013
@@ -47,6 +47,18 @@ New features
http://blog.mikemccandless.com/2013/08/suggeststopfilter-carefully-removes.html
for details.
+* LUCENE-5178: Added support for missing values to DocValues fields.
+ AtomicReader.getDocsWithField returns a Bits of documents with a value,
+ and FieldCache.getDocsWithField forwards to that for DocValues fields. Things like
+ SortField.setMissingValue, FunctionValues.exists, and FieldValueFilter now
+ work with DocValues fields. (Robert Muir)
+
+* LUCENE-5124: Lucene 4.5 has a new Lucene45Codec with Lucene45DocValues,
+ supporting missing values and with most datastructures residing off-heap.
+ Added "Memory" docvalues format that works entirely in heap, and "Disk"
+ loads no datastructures into RAM. Both of these also support missing values.
+ Added DiskNormsFormat (in case you want norms entirely on disk). (Robert Muir)
+
Bug Fixes
* LUCENE-5116: IndexWriter.addIndexes(IndexReader...) should drop empty (or all
@@ -142,6 +154,15 @@ API Changes
* LUCENE-5173: Lucene never writes segments with 0 documents anymore.
(Shai Erera, Uwe Schindler, Robert Muir)
+* LUCENE-5178: SortedDocValues always returns -1 ord when a document is missing
+ a value for the field. Previously it only did this if the SortedDocValues
+ was produced by uninversion on the FieldCache. (Robert Muir)
+
+Changes in Runtime Behavior
+
+* LUCENE-5178: DocValues codec consumer APIs (iterables) return null values
+ when the document has no value for the field. (Robert Muir)
+
Optimizations
* LUCENE-5088: Added TermFilter to filter docs by a specific term.
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java Tue Aug 20 22:47:30 2013
@@ -22,8 +22,11 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.lucene45.Lucene45DocValuesConsumer;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.util.BytesRef;
/**
* DocValues format that keeps most things on disk.
@@ -40,7 +43,12 @@ public final class DiskDocValuesFormat e
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return new DiskDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+ return new Lucene45DocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION) {
+ @Override
+ protected void addTermsDict(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ addBinaryField(field, values);
+ }
+ };
}
@Override
@@ -52,11 +60,4 @@ public final class DiskDocValuesFormat e
public static final String DATA_EXTENSION = "dvdd";
public static final String META_CODEC = "DiskDocValuesMetadata";
public static final String META_EXTENSION = "dvdm";
- public static final int VERSION_START = 0;
- public static final int VERSION_COMPRESSED_TERMS = 1;
- public static final int VERSION_CURRENT = VERSION_COMPRESSED_TERMS;
- public static final byte NUMERIC = 0;
- public static final byte BINARY = 1;
- public static final byte SORTED = 2;
- public static final byte SORTED_SET = 3;
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java Tue Aug 20 22:47:30 2013
@@ -17,726 +17,34 @@ package org.apache.lucene.codecs.diskdv;
* limitations under the License.
*/
-import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.DELTA_COMPRESSED;
-import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.GCD_COMPRESSED;
-import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.TABLE_COMPRESSED;
-
-import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.BINARY_FIXED_UNCOMPRESSED;
-import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED;
-import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.BINARY_PREFIX_COMPRESSED;
-
import java.io.IOException;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.DocValuesProducer;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.DocsAndPositionsEnum;
-import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.codecs.lucene45.Lucene45DocValuesProducer;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.packed.BlockPackedReader;
import org.apache.lucene.util.packed.MonotonicBlockPackedReader;
-import org.apache.lucene.util.packed.PackedInts;
-class DiskDocValuesProducer extends DocValuesProducer {
- private final Map<Integer,NumericEntry> numerics;
- private final Map<Integer,BinaryEntry> binaries;
- private final Map<Integer,NumericEntry> ords;
- private final Map<Integer,NumericEntry> ordIndexes;
- private final IndexInput data;
+class DiskDocValuesProducer extends Lucene45DocValuesProducer {
- // memory-resident structures
- private final Map<Integer,MonotonicBlockPackedReader> addressInstances = new HashMap<Integer,MonotonicBlockPackedReader>();
- private final Map<Integer,MonotonicBlockPackedReader> ordIndexInstances = new HashMap<Integer,MonotonicBlockPackedReader>();
-
DiskDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
- String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
- // read in the entries from the metadata file.
- IndexInput in = state.directory.openInput(metaName, state.context);
- boolean success = false;
- final int version;
- try {
- version = CodecUtil.checkHeader(in, metaCodec,
- DiskDocValuesFormat.VERSION_CURRENT,
- DiskDocValuesFormat.VERSION_CURRENT);
- numerics = new HashMap<Integer,NumericEntry>();
- ords = new HashMap<Integer,NumericEntry>();
- ordIndexes = new HashMap<Integer,NumericEntry>();
- binaries = new HashMap<Integer,BinaryEntry>();
- readFields(in, state.fieldInfos);
-
- success = true;
- } finally {
- if (success) {
- IOUtils.close(in);
- } else {
- IOUtils.closeWhileHandlingException(in);
- }
- }
-
- success = false;
- try {
- String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
- data = state.directory.openInput(dataName, state.context);
- final int version2 = CodecUtil.checkHeader(data, dataCodec,
- DiskDocValuesFormat.VERSION_CURRENT,
- DiskDocValuesFormat.VERSION_CURRENT);
- if (version != version2) {
- throw new CorruptIndexException("Format versions mismatch");
- }
-
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(this.data);
- }
- }
- }
-
- private void readFields(IndexInput meta, FieldInfos infos) throws IOException {
- int fieldNumber = meta.readVInt();
- while (fieldNumber != -1) {
- byte type = meta.readByte();
- if (type == DiskDocValuesFormat.NUMERIC) {
- numerics.put(fieldNumber, readNumericEntry(meta));
- } else if (type == DiskDocValuesFormat.BINARY) {
- BinaryEntry b = readBinaryEntry(meta);
- binaries.put(fieldNumber, b);
- } else if (type == DiskDocValuesFormat.SORTED) {
- // sorted = binary + numeric
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- if (meta.readByte() != DiskDocValuesFormat.BINARY) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- BinaryEntry b = readBinaryEntry(meta);
- binaries.put(fieldNumber, b);
-
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- if (meta.readByte() != DiskDocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sorted entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- NumericEntry n = readNumericEntry(meta);
- ords.put(fieldNumber, n);
- } else if (type == DiskDocValuesFormat.SORTED_SET) {
- // sortedset = binary + numeric + ordIndex
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- if (meta.readByte() != DiskDocValuesFormat.BINARY) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- BinaryEntry b = readBinaryEntry(meta);
- binaries.put(fieldNumber, b);
-
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- if (meta.readByte() != DiskDocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- NumericEntry n1 = readNumericEntry(meta);
- ords.put(fieldNumber, n1);
-
- if (meta.readVInt() != fieldNumber) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- if (meta.readByte() != DiskDocValuesFormat.NUMERIC) {
- throw new CorruptIndexException("sortedset entry for field: " + fieldNumber + " is corrupt (resource=" + meta + ")");
- }
- NumericEntry n2 = readNumericEntry(meta);
- ordIndexes.put(fieldNumber, n2);
- } else {
- throw new CorruptIndexException("invalid type: " + type + ", resource=" + meta);
- }
- fieldNumber = meta.readVInt();
- }
- }
-
- static NumericEntry readNumericEntry(IndexInput meta) throws IOException {
- NumericEntry entry = new NumericEntry();
- entry.format = meta.readVInt();
- entry.packedIntsVersion = meta.readVInt();
- entry.offset = meta.readLong();
- entry.count = meta.readVLong();
- entry.blockSize = meta.readVInt();
- switch(entry.format) {
- case GCD_COMPRESSED:
- entry.minValue = meta.readLong();
- entry.gcd = meta.readLong();
- break;
- case TABLE_COMPRESSED:
- if (entry.count > Integer.MAX_VALUE) {
- throw new CorruptIndexException("Cannot use TABLE_COMPRESSED with more than MAX_VALUE values, input=" + meta);
- }
- final int uniqueValues = meta.readVInt();
- if (uniqueValues > 256) {
- throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values, input=" + meta);
- }
- entry.table = new long[uniqueValues];
- for (int i = 0; i < uniqueValues; ++i) {
- entry.table[i] = meta.readLong();
- }
- break;
- case DELTA_COMPRESSED:
- break;
- default:
- throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
- }
- return entry;
- }
-
- static BinaryEntry readBinaryEntry(IndexInput meta) throws IOException {
- BinaryEntry entry = new BinaryEntry();
- entry.format = meta.readVInt();
- entry.minLength = meta.readVInt();
- entry.maxLength = meta.readVInt();
- entry.count = meta.readVLong();
- entry.offset = meta.readLong();
- switch(entry.format) {
- case BINARY_FIXED_UNCOMPRESSED:
- break;
- case BINARY_PREFIX_COMPRESSED:
- entry.addressInterval = meta.readVInt();
- entry.addressesOffset = meta.readLong();
- entry.packedIntsVersion = meta.readVInt();
- entry.blockSize = meta.readVInt();
- break;
- case BINARY_VARIABLE_UNCOMPRESSED:
- entry.addressesOffset = meta.readLong();
- entry.packedIntsVersion = meta.readVInt();
- entry.blockSize = meta.readVInt();
- break;
- default:
- throw new CorruptIndexException("Unknown format: " + entry.format + ", input=" + meta);
- }
- return entry;
+ super(state, dataCodec, dataExtension, metaCodec, metaExtension);
}
@Override
- public NumericDocValues getNumeric(FieldInfo field) throws IOException {
- NumericEntry entry = numerics.get(field.number);
- return getNumeric(entry);
- }
-
- LongNumericDocValues getNumeric(NumericEntry entry) throws IOException {
- final IndexInput data = this.data.clone();
- data.seek(entry.offset);
-
- switch (entry.format) {
- case DELTA_COMPRESSED:
- final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true);
- return new LongNumericDocValues() {
- @Override
- public long get(long id) {
- return reader.get(id);
- }
- };
- case GCD_COMPRESSED:
- final long min = entry.minValue;
- final long mult = entry.gcd;
- final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true);
- return new LongNumericDocValues() {
- @Override
- public long get(long id) {
- return min + mult * quotientReader.get(id);
- }
- };
- case TABLE_COMPRESSED:
- final long table[] = entry.table;
- final int bitsRequired = PackedInts.bitsRequired(table.length - 1);
- final PackedInts.Reader ords = PackedInts.getDirectReaderNoHeader(data, PackedInts.Format.PACKED, entry.packedIntsVersion, (int) entry.count, bitsRequired);
- return new LongNumericDocValues() {
- @Override
- public long get(long id) {
- return table[(int) ords.get((int) id)];
- }
- };
- default:
- throw new AssertionError();
- }
+ protected MonotonicBlockPackedReader getAddressInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ data.seek(bytes.addressesOffset);
+ return new MonotonicBlockPackedReader(data.clone(), bytes.packedIntsVersion, bytes.blockSize, bytes.count, true);
}
@Override
- public BinaryDocValues getBinary(FieldInfo field) throws IOException {
- BinaryEntry bytes = binaries.get(field.number);
- switch(bytes.format) {
- case BINARY_FIXED_UNCOMPRESSED:
- return getFixedBinary(field, bytes);
- case BINARY_VARIABLE_UNCOMPRESSED:
- return getVariableBinary(field, bytes);
- case BINARY_PREFIX_COMPRESSED:
- return getCompressedBinary(field, bytes);
- default:
- throw new AssertionError();
- }
- }
-
- private BinaryDocValues getFixedBinary(FieldInfo field, final BinaryEntry bytes) {
- final IndexInput data = this.data.clone();
-
- return new LongBinaryDocValues() {
- @Override
- public void get(long id, BytesRef result) {
- long address = bytes.offset + id * bytes.maxLength;
- try {
- data.seek(address);
- // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
- // assume "they" own the bytes after calling this!
- final byte[] buffer = new byte[bytes.maxLength];
- data.readBytes(buffer, 0, buffer.length);
- result.bytes = buffer;
- result.offset = 0;
- result.length = buffer.length;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- };
- }
-
- private BinaryDocValues getVariableBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
- final IndexInput data = this.data.clone();
-
- final MonotonicBlockPackedReader addresses;
- synchronized (addressInstances) {
- MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number);
- if (addrInstance == null) {
- data.seek(bytes.addressesOffset);
- addrInstance = new MonotonicBlockPackedReader(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count, false);
- addressInstances.put(field.number, addrInstance);
- }
- addresses = addrInstance;
- }
-
- return new LongBinaryDocValues() {
- @Override
- public void get(long id, BytesRef result) {
- long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1));
- long endAddress = bytes.offset + addresses.get(id);
- int length = (int) (endAddress - startAddress);
- try {
- data.seek(startAddress);
- // NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
- // assume "they" own the bytes after calling this!
- final byte[] buffer = new byte[length];
- data.readBytes(buffer, 0, buffer.length);
- result.bytes = buffer;
- result.offset = 0;
- result.length = length;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- };
- }
-
- private BinaryDocValues getCompressedBinary(FieldInfo field, final BinaryEntry bytes) throws IOException {
- final IndexInput data = this.data.clone();
- final long interval = bytes.addressInterval;
-
- final MonotonicBlockPackedReader addresses;
- synchronized (addressInstances) {
- MonotonicBlockPackedReader addrInstance = addressInstances.get(field.number);
- if (addrInstance == null) {
- data.seek(bytes.addressesOffset);
- final long size;
- if (bytes.count % interval == 0) {
- size = bytes.count / interval;
- } else {
- size = 1L + bytes.count / interval;
- }
- addrInstance = new MonotonicBlockPackedReader(data, bytes.packedIntsVersion, bytes.blockSize, size, false);
- addressInstances.put(field.number, addrInstance);
- }
- addresses = addrInstance;
- }
-
- return new CompressedBinaryDocValues(bytes, addresses, data);
+ protected MonotonicBlockPackedReader getIntervalInstance(IndexInput data, FieldInfo field, BinaryEntry bytes) throws IOException {
+ throw new AssertionError();
}
@Override
- public SortedDocValues getSorted(FieldInfo field) throws IOException {
- final int valueCount = (int) binaries.get(field.number).count;
- final BinaryDocValues binary = getBinary(field);
- NumericEntry entry = ords.get(field.number);
- IndexInput data = this.data.clone();
+ protected MonotonicBlockPackedReader getOrdIndexInstance(IndexInput data, FieldInfo field, NumericEntry entry) throws IOException {
data.seek(entry.offset);
- final BlockPackedReader ordinals = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true);
-
- return new SortedDocValues() {
-
- @Override
- public int getOrd(int docID) {
- return (int) ordinals.get(docID);
- }
-
- @Override
- public void lookupOrd(int ord, BytesRef result) {
- binary.get(ord, result);
- }
-
- @Override
- public int getValueCount() {
- return valueCount;
- }
-
- @Override
- public int lookupTerm(BytesRef key) {
- if (binary instanceof CompressedBinaryDocValues) {
- return (int) ((CompressedBinaryDocValues)binary).lookupTerm(key);
- } else {
- return super.lookupTerm(key);
- }
- }
-
- @Override
- public TermsEnum termsEnum() {
- if (binary instanceof CompressedBinaryDocValues) {
- return ((CompressedBinaryDocValues)binary).getTermsEnum();
- } else {
- return super.termsEnum();
- }
- }
- };
- }
-
- @Override
- public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
- final long valueCount = binaries.get(field.number).count;
- // we keep the byte[]s and list of ords on disk, these could be large
- final LongBinaryDocValues binary = (LongBinaryDocValues) getBinary(field);
- final LongNumericDocValues ordinals = getNumeric(ords.get(field.number));
- // but the addresses to the ord stream are in RAM
- final MonotonicBlockPackedReader ordIndex;
- synchronized (ordIndexInstances) {
- MonotonicBlockPackedReader ordIndexInstance = ordIndexInstances.get(field.number);
- if (ordIndexInstance == null) {
- NumericEntry entry = ordIndexes.get(field.number);
- IndexInput data = this.data.clone();
- data.seek(entry.offset);
- ordIndexInstance = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, false);
- ordIndexInstances.put(field.number, ordIndexInstance);
- }
- ordIndex = ordIndexInstance;
- }
-
- return new SortedSetDocValues() {
- long offset;
- long endOffset;
-
- @Override
- public long nextOrd() {
- if (offset == endOffset) {
- return NO_MORE_ORDS;
- } else {
- long ord = ordinals.get(offset);
- offset++;
- return ord;
- }
- }
-
- @Override
- public void setDocument(int docID) {
- offset = (docID == 0 ? 0 : ordIndex.get(docID-1));
- endOffset = ordIndex.get(docID);
- }
-
- @Override
- public void lookupOrd(long ord, BytesRef result) {
- binary.get(ord, result);
- }
-
- @Override
- public long getValueCount() {
- return valueCount;
- }
-
- @Override
- public long lookupTerm(BytesRef key) {
- if (binary instanceof CompressedBinaryDocValues) {
- return ((CompressedBinaryDocValues)binary).lookupTerm(key);
- } else {
- return super.lookupTerm(key);
- }
- }
-
- @Override
- public TermsEnum termsEnum() {
- if (binary instanceof CompressedBinaryDocValues) {
- return ((CompressedBinaryDocValues)binary).getTermsEnum();
- } else {
- return super.termsEnum();
- }
- }
- };
- }
-
- @Override
- public void close() throws IOException {
- data.close();
- }
-
- static class NumericEntry {
- long offset;
-
- int format;
- int packedIntsVersion;
- long count;
- int blockSize;
-
- long minValue;
- long gcd;
- long table[];
- }
-
- static class BinaryEntry {
- long offset;
-
- int format;
- long count;
- int minLength;
- int maxLength;
- long addressesOffset;
- long addressInterval;
- int packedIntsVersion;
- int blockSize;
- }
-
- // internally we compose complex dv (sorted/sortedset) from other ones
- static abstract class LongNumericDocValues extends NumericDocValues {
- @Override
- public final long get(int docID) {
- return get((long) docID);
- }
-
- abstract long get(long id);
- }
-
- static abstract class LongBinaryDocValues extends BinaryDocValues {
- @Override
- public final void get(int docID, BytesRef result) {
- get((long)docID, result);
- }
-
- abstract void get(long id, BytesRef Result);
- }
-
- // in the compressed case, we add a few additional operations for
- // more efficient reverse lookup and enumeration
- static class CompressedBinaryDocValues extends LongBinaryDocValues {
- final BinaryEntry bytes;
- final long interval;
- final long numValues;
- final long numIndexValues;
- final MonotonicBlockPackedReader addresses;
- final IndexInput data;
- final TermsEnum termsEnum;
-
- public CompressedBinaryDocValues(BinaryEntry bytes, MonotonicBlockPackedReader addresses, IndexInput data) throws IOException {
- this.bytes = bytes;
- this.interval = bytes.addressInterval;
- this.addresses = addresses;
- this.data = data;
- this.numValues = bytes.count;
- this.numIndexValues = addresses.size();
- this.termsEnum = getTermsEnum(data);
- }
-
- @Override
- public void get(long id, BytesRef result) {
- try {
- termsEnum.seekExact(id);
- BytesRef term = termsEnum.term();
- result.bytes = term.bytes;
- result.offset = term.offset;
- result.length = term.length;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- long lookupTerm(BytesRef key) {
- try {
- SeekStatus status = termsEnum.seekCeil(key);
- if (status == SeekStatus.END) {
- return -numValues-1;
- } else if (status == SeekStatus.FOUND) {
- return termsEnum.ord();
- } else {
- return -termsEnum.ord()-1;
- }
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
- }
-
- TermsEnum getTermsEnum() {
- try {
- return getTermsEnum(data.clone());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private TermsEnum getTermsEnum(final IndexInput input) throws IOException {
- input.seek(bytes.offset);
-
- return new TermsEnum() {
- private long currentOrd = -1;
- // TODO: maxLength is negative when all terms are merged away...
- private final BytesRef termBuffer = new BytesRef(bytes.maxLength < 0 ? 0 : bytes.maxLength);
- private final BytesRef term = new BytesRef(); // TODO: paranoia?
-
- @Override
- public BytesRef next() throws IOException {
- if (doNext() == null) {
- return null;
- } else {
- setTerm();
- return term;
- }
- }
-
- private BytesRef doNext() throws IOException {
- if (++currentOrd >= numValues) {
- return null;
- } else {
- int start = input.readVInt();
- int suffix = input.readVInt();
- input.readBytes(termBuffer.bytes, start, suffix);
- termBuffer.length = start + suffix;
- return termBuffer;
- }
- }
-
- @Override
- public SeekStatus seekCeil(BytesRef text) throws IOException {
- // binary-search just the index values to find the block,
- // then scan within the block
- long low = 0;
- long high = numIndexValues-1;
-
- while (low <= high) {
- long mid = (low + high) >>> 1;
- doSeek(mid * interval);
- int cmp = termBuffer.compareTo(text);
-
- if (cmp < 0) {
- low = mid + 1;
- } else if (cmp > 0) {
- high = mid - 1;
- } else {
- // we got lucky, found an indexed term
- setTerm();
- return SeekStatus.FOUND;
- }
- }
-
- if (numIndexValues == 0) {
- return SeekStatus.END;
- }
-
- // block before insertion point
- long block = low-1;
- doSeek(block < 0 ? -1 : block * interval);
-
- while (doNext() != null) {
- int cmp = termBuffer.compareTo(text);
- if (cmp == 0) {
- setTerm();
- return SeekStatus.FOUND;
- } else if (cmp > 0) {
- setTerm();
- return SeekStatus.NOT_FOUND;
- }
- }
-
- return SeekStatus.END;
- }
-
- @Override
- public void seekExact(long ord) throws IOException {
- doSeek(ord);
- setTerm();
- }
-
- private void doSeek(long ord) throws IOException {
- long block = ord / interval;
-
- if (ord >= currentOrd && block == currentOrd / interval) {
- // seek within current block
- } else {
- // position before start of block
- currentOrd = ord - ord % interval - 1;
- input.seek(bytes.offset + addresses.get(block));
- }
-
- while (currentOrd < ord) {
- doNext();
- }
- }
-
- private void setTerm() {
- // TODO: is there a cleaner way
- term.bytes = new byte[termBuffer.length];
- term.offset = 0;
- term.copyBytes(termBuffer);
- }
-
- @Override
- public BytesRef term() throws IOException {
- return term;
- }
-
- @Override
- public long ord() throws IOException {
- return currentOrd;
- }
-
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public int docFreq() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public long totalTermFreq() throws IOException {
- return -1;
- }
-
- @Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
- throw new UnsupportedOperationException();
- }
- };
- }
+ return new MonotonicBlockPackedReader(data.clone(), entry.packedIntsVersion, entry.blockSize, entry.count, true);
}
}
Copied: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (from r1515977, lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java?p2=lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java&p1=lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java&r1=1515977&r2=1516012&rev=1516012&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java Tue Aug 20 22:47:30 2013
@@ -98,7 +98,7 @@ class MemoryDocValuesConsumer extends Do
// TODO: more efficient?
HashSet<Long> uniqueValues = null;
if (optimizeStorage) {
- uniqueValues = new HashSet<>();
+ uniqueValues = new HashSet<Long>();
long count = 0;
for (Number nv : values) {
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/package.html?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/package.html (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/package.html Tue Aug 20 22:47:30 2013
@@ -20,6 +20,6 @@
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
-Postings format that is read entirely into memory.
+Postings and DocValues formats that are read entirely into memory.
</body>
</html>
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesFormat.java Tue Aug 20 22:47:30 2013
@@ -38,12 +38,16 @@ import org.apache.lucene.index.SegmentWr
* minvalue 0
* pattern 000
* 005
+ * T
* 234
+ * T
* 123
+ * T
* ...
* </pre>
* so a document's value (delta encoded from minvalue) can be retrieved by
- * seeking to startOffset + (1+pattern.length())*docid. The extra 1 is the newline.
+ * seeking to startOffset + (1+pattern.length()+2)*docid. The extra 1 is the newline.
+ * The extra 2 is another newline and 'T' or 'F': true if the value is real, false if missing.
*
* for bytes this is also a "fixed-width" file, for example:
* <pre>
@@ -53,12 +57,15 @@ import org.apache.lucene.index.SegmentWr
* pattern 0
* length 6
* foobar[space][space]
+ * T
* length 3
* baz[space][space][space][space][space]
+ * T
* ...
* </pre>
- * so a doc's value can be retrieved by seeking to startOffset + (9+pattern.length+maxlength)*doc
+ * so a doc's value can be retrieved by seeking to startOffset + (9+pattern.length+maxlength+2)*doc
* the extra 9 is 2 newlines, plus "length " itself.
+ * the extra 2 is another newline and 'T' or 'F': true if the value is real, false if missing.
*
* for sorted bytes this is a fixed-width file, for example:
* <pre>
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java Tue Aug 20 22:47:30 2013
@@ -38,6 +38,7 @@ import org.apache.lucene.index.SortedDoc
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper;
@@ -100,7 +101,7 @@ class SimpleTextDocValuesReader extends
assert startsWith(PATTERN);
field.pattern = stripPrefix(PATTERN);
field.dataStartFilePointer = data.getFilePointer();
- data.seek(data.getFilePointer() + (1+field.pattern.length()) * maxDoc);
+ data.seek(data.getFilePointer() + (1+field.pattern.length()+2) * maxDoc);
} else if (dvType == DocValuesType.BINARY) {
readLine();
assert startsWith(MAXLENGTH);
@@ -109,7 +110,7 @@ class SimpleTextDocValuesReader extends
assert startsWith(PATTERN);
field.pattern = stripPrefix(PATTERN);
field.dataStartFilePointer = data.getFilePointer();
- data.seek(data.getFilePointer() + (9+field.pattern.length()+field.maxLength) * maxDoc);
+ data.seek(data.getFilePointer() + (9+field.pattern.length()+field.maxLength+2) * maxDoc);
} else if (dvType == DocValuesType.SORTED || dvType == DocValuesType.SORTED_SET) {
readLine();
assert startsWith(NUMVALUES);
@@ -158,7 +159,7 @@ class SimpleTextDocValuesReader extends
if (docID < 0 || docID >= maxDoc) {
throw new IndexOutOfBoundsException("docID must be 0 .. " + (maxDoc-1) + "; got " + docID);
}
- in.seek(field.dataStartFilePointer + (1+field.pattern.length())*docID);
+ in.seek(field.dataStartFilePointer + (1+field.pattern.length()+2)*docID);
SimpleTextUtil.readLine(in, scratch);
//System.out.println("parsing delta: " + scratch.utf8ToString());
BigDecimal bd;
@@ -169,6 +170,7 @@ class SimpleTextDocValuesReader extends
e.initCause(pe);
throw e;
}
+ SimpleTextUtil.readLine(in, scratch); // read the line telling us if its real or not
return BigInteger.valueOf(field.minValue).add(bd.toBigIntegerExact()).longValue();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
@@ -176,6 +178,30 @@ class SimpleTextDocValuesReader extends
}
};
}
+
+ private Bits getNumericDocsWithField(FieldInfo fieldInfo) throws IOException {
+ final OneField field = fields.get(fieldInfo.name);
+ final IndexInput in = data.clone();
+ final BytesRef scratch = new BytesRef();
+ return new Bits() {
+ @Override
+ public boolean get(int index) {
+ try {
+ in.seek(field.dataStartFilePointer + (1+field.pattern.length()+2)*index);
+ SimpleTextUtil.readLine(in, scratch); // data
+ SimpleTextUtil.readLine(in, scratch); // 'T' or 'F'
+ return scratch.bytes[scratch.offset] == (byte) 'T';
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ };
+ }
@Override
public BinaryDocValues getBinary(FieldInfo fieldInfo) throws IOException {
@@ -196,7 +222,7 @@ class SimpleTextDocValuesReader extends
if (docID < 0 || docID >= maxDoc) {
throw new IndexOutOfBoundsException("docID must be 0 .. " + (maxDoc-1) + "; got " + docID);
}
- in.seek(field.dataStartFilePointer + (9+field.pattern.length() + field.maxLength)*docID);
+ in.seek(field.dataStartFilePointer + (9+field.pattern.length() + field.maxLength+2)*docID);
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch, LENGTH);
int len;
@@ -217,6 +243,45 @@ class SimpleTextDocValuesReader extends
}
};
}
+
+ private Bits getBinaryDocsWithField(FieldInfo fieldInfo) throws IOException {
+ final OneField field = fields.get(fieldInfo.name);
+ final IndexInput in = data.clone();
+ final BytesRef scratch = new BytesRef();
+ final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
+
+ return new Bits() {
+ @Override
+ public boolean get(int index) {
+ try {
+ in.seek(field.dataStartFilePointer + (9+field.pattern.length() + field.maxLength+2)*index);
+ SimpleTextUtil.readLine(in, scratch);
+ assert StringHelper.startsWith(scratch, LENGTH);
+ int len;
+ try {
+ len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue();
+ } catch (ParseException pe) {
+ CorruptIndexException e = new CorruptIndexException("failed to parse int length (resource=" + in + ")");
+ e.initCause(pe);
+ throw e;
+ }
+ // skip past bytes
+ byte bytes[] = new byte[len];
+ in.readBytes(bytes, 0, len);
+ SimpleTextUtil.readLine(in, scratch); // newline
+ SimpleTextUtil.readLine(in, scratch); // 'T' or 'F'
+ return scratch.bytes[scratch.offset] == (byte) 'T';
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ };
+ }
@Override
public SortedDocValues getSorted(FieldInfo fieldInfo) throws IOException {
@@ -241,7 +306,7 @@ class SimpleTextDocValuesReader extends
in.seek(field.dataStartFilePointer + field.numValues * (9 + field.pattern.length() + field.maxLength) + docID * (1 + field.ordPattern.length()));
SimpleTextUtil.readLine(in, scratch);
try {
- return ordDecoder.parse(scratch.utf8ToString()).intValue();
+ return (int) ordDecoder.parse(scratch.utf8ToString()).longValue()-1;
} catch (ParseException pe) {
CorruptIndexException e = new CorruptIndexException("failed to parse ord (resource=" + in + ")");
e.initCause(pe);
@@ -362,6 +427,22 @@ class SimpleTextDocValuesReader extends
}
};
}
+
+ @Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ switch (field.getDocValuesType()) {
+ case SORTED_SET:
+ return new SortedSetDocsWithField(getSortedSet(field), maxDoc);
+ case SORTED:
+ return new SortedDocsWithField(getSorted(field), maxDoc);
+ case BINARY:
+ return getBinaryDocsWithField(field);
+ case NUMERIC:
+ return getNumericDocsWithField(field);
+ default:
+ throw new AssertionError();
+ }
+ }
@Override
public void close() throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java Tue Aug 20 22:47:30 2013
@@ -78,7 +78,7 @@ class SimpleTextDocValuesWriter extends
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
for(Number n : values) {
- long v = n.longValue();
+ long v = n == null ? 0 : n.longValue();
minValue = Math.min(minValue, v);
maxValue = Math.max(maxValue, v);
}
@@ -112,13 +112,19 @@ class SimpleTextDocValuesWriter extends
// second pass to write the values
for(Number n : values) {
- long value = n.longValue();
+ long value = n == null ? 0 : n.longValue();
assert value >= minValue;
Number delta = BigInteger.valueOf(value).subtract(BigInteger.valueOf(minValue));
String s = encoder.format(delta);
assert s.length() == patternString.length();
SimpleTextUtil.write(data, s, scratch);
SimpleTextUtil.writeNewline(data);
+ if (n == null) {
+ SimpleTextUtil.write(data, "F", scratch);
+ } else {
+ SimpleTextUtil.write(data, "T", scratch);
+ }
+ SimpleTextUtil.writeNewline(data);
numDocsWritten++;
assert numDocsWritten <= numDocs;
}
@@ -132,7 +138,8 @@ class SimpleTextDocValuesWriter extends
assert field.getDocValuesType() == DocValuesType.BINARY;
int maxLength = 0;
for(BytesRef value : values) {
- maxLength = Math.max(maxLength, value.length);
+ final int length = value == null ? 0 : value.length;
+ maxLength = Math.max(maxLength, length);
}
writeFieldEntry(field, FieldInfo.DocValuesType.BINARY);
@@ -155,19 +162,28 @@ class SimpleTextDocValuesWriter extends
int numDocsWritten = 0;
for(BytesRef value : values) {
// write length
+ final int length = value == null ? 0 : value.length;
SimpleTextUtil.write(data, LENGTH);
- SimpleTextUtil.write(data, encoder.format(value.length), scratch);
+ SimpleTextUtil.write(data, encoder.format(length), scratch);
SimpleTextUtil.writeNewline(data);
// write bytes -- don't use SimpleText.write
// because it escapes:
- data.writeBytes(value.bytes, value.offset, value.length);
+ if (value != null) {
+ data.writeBytes(value.bytes, value.offset, value.length);
+ }
// pad to fit
- for (int i = value.length; i < maxLength; i++) {
+ for (int i = length; i < maxLength; i++) {
data.writeByte((byte)' ');
}
SimpleTextUtil.writeNewline(data);
+ if (value == null) {
+ SimpleTextUtil.write(data, "F", scratch);
+ } else {
+ SimpleTextUtil.write(data, "T", scratch);
+ }
+ SimpleTextUtil.writeNewline(data);
numDocsWritten++;
}
@@ -209,7 +225,7 @@ class SimpleTextDocValuesWriter extends
SimpleTextUtil.writeNewline(data);
final DecimalFormat encoder = new DecimalFormat(sb.toString(), new DecimalFormatSymbols(Locale.ROOT));
- int maxOrdBytes = Integer.toString(valueCount).length();
+ int maxOrdBytes = Long.toString(valueCount+1L).length();
sb.setLength(0);
for (int i = 0; i < maxOrdBytes; i++) {
sb.append('0');
@@ -246,7 +262,7 @@ class SimpleTextDocValuesWriter extends
assert valuesSeen == valueCount;
for(Number ord : docToOrd) {
- SimpleTextUtil.write(data, ordEncoder.format(ord.intValue()), scratch);
+ SimpleTextUtil.write(data, ordEncoder.format(ord.longValue()+1), scratch);
SimpleTextUtil.writeNewline(data);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.DocValuesFormat Tue Aug 20 22:47:30 2013
@@ -14,4 +14,5 @@
# limitations under the License.
org.apache.lucene.codecs.diskdv.DiskDocValuesFormat
+org.apache.lucene.codecs.memory.MemoryDocValuesFormat
org.apache.lucene.codecs.simpletext.SimpleTextDocValuesFormat
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/Codec.java Tue Aug 20 22:47:30 2013
@@ -119,7 +119,7 @@ public abstract class Codec implements N
loader.reload(classloader);
}
- private static Codec defaultCodec = Codec.forName("Lucene42");
+ private static Codec defaultCodec = Codec.forName("Lucene45");
/** expert: returns the default codec used for newly created
* {@link IndexWriterConfig}s.
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java Tue Aug 20 22:47:30 2013
@@ -69,7 +69,8 @@ public abstract class DocValuesConsumer
/**
* Writes numeric docvalues for a field.
* @param field field information
- * @param values Iterable of numeric values (one for each document).
+ * @param values Iterable of numeric values (one for each document). {@code null} indicates
+ * a missing value.
* @throws IOException if an I/O error occurred.
*/
public abstract void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException;
@@ -77,7 +78,8 @@ public abstract class DocValuesConsumer
/**
* Writes binary docvalues for a field.
* @param field field information
- * @param values Iterable of binary values (one for each document).
+ * @param values Iterable of binary values (one for each document). {@code null} indicates
+ * a missing value.
* @throws IOException if an I/O error occurred.
*/
public abstract void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException;
@@ -86,7 +88,8 @@ public abstract class DocValuesConsumer
* Writes pre-sorted binary docvalues for a field.
* @param field field information
* @param values Iterable of binary values in sorted order (deduplicated).
- * @param docToOrd Iterable of ordinals (one for each document).
+ * @param docToOrd Iterable of ordinals (one for each document). {@code -1} indicates
+ * a missing value.
* @throws IOException if an I/O error occurred.
*/
public abstract void addSortedField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrd) throws IOException;
@@ -95,7 +98,8 @@ public abstract class DocValuesConsumer
* Writes pre-sorted set docvalues for a field
* @param field field information
* @param values Iterable of binary values in sorted order (deduplicated).
- * @param docToOrdCount Iterable of the number of values for each document.
+ * @param docToOrdCount Iterable of the number of values for each document. A zero ordinal
+ * count indicates a missing value.
* @param ords Iterable of ordinal occurrences (docToOrdCount*maxDoc total).
* @throws IOException if an I/O error occurred.
*/
@@ -107,7 +111,7 @@ public abstract class DocValuesConsumer
* The default implementation calls {@link #addNumericField}, passing
* an Iterable that merges and filters deleted documents on the fly.
*/
- public void mergeNumericField(FieldInfo fieldInfo, final MergeState mergeState, final List<NumericDocValues> toMerge) throws IOException {
+ public void mergeNumericField(final FieldInfo fieldInfo, final MergeState mergeState, final List<NumericDocValues> toMerge, final List<Bits> docsWithField) throws IOException {
addNumericField(fieldInfo,
new Iterable<Number>() {
@@ -116,10 +120,11 @@ public abstract class DocValuesConsumer
return new Iterator<Number>() {
int readerUpto = -1;
int docIDUpto;
- long nextValue;
+ Long nextValue;
AtomicReader currentReader;
NumericDocValues currentValues;
Bits currentLiveDocs;
+ Bits currentDocsWithField;
boolean nextIsSet;
@Override
@@ -139,7 +144,6 @@ public abstract class DocValuesConsumer
}
assert nextIsSet;
nextIsSet = false;
- // TODO: make a mutable number
return nextValue;
}
@@ -155,6 +159,7 @@ public abstract class DocValuesConsumer
currentReader = mergeState.readers.get(readerUpto);
currentValues = toMerge.get(readerUpto);
currentLiveDocs = currentReader.getLiveDocs();
+ currentDocsWithField = docsWithField.get(readerUpto);
}
docIDUpto = 0;
continue;
@@ -162,7 +167,11 @@ public abstract class DocValuesConsumer
if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
nextIsSet = true;
- nextValue = currentValues.get(docIDUpto);
+ if (currentDocsWithField.get(docIDUpto)) {
+ nextValue = currentValues.get(docIDUpto);
+ } else {
+ nextValue = null;
+ }
docIDUpto++;
return true;
}
@@ -181,7 +190,7 @@ public abstract class DocValuesConsumer
* The default implementation calls {@link #addBinaryField}, passing
* an Iterable that merges and filters deleted documents on the fly.
*/
- public void mergeBinaryField(FieldInfo fieldInfo, final MergeState mergeState, final List<BinaryDocValues> toMerge) throws IOException {
+ public void mergeBinaryField(FieldInfo fieldInfo, final MergeState mergeState, final List<BinaryDocValues> toMerge, final List<Bits> docsWithField) throws IOException {
addBinaryField(fieldInfo,
new Iterable<BytesRef>() {
@@ -191,9 +200,11 @@ public abstract class DocValuesConsumer
int readerUpto = -1;
int docIDUpto;
BytesRef nextValue = new BytesRef();
+ BytesRef nextPointer; // points to null if missing, or nextValue
AtomicReader currentReader;
BinaryDocValues currentValues;
Bits currentLiveDocs;
+ Bits currentDocsWithField;
boolean nextIsSet;
@Override
@@ -213,8 +224,7 @@ public abstract class DocValuesConsumer
}
assert nextIsSet;
nextIsSet = false;
- // TODO: make a mutable number
- return nextValue;
+ return nextPointer;
}
private boolean setNext() {
@@ -228,6 +238,7 @@ public abstract class DocValuesConsumer
if (readerUpto < toMerge.size()) {
currentReader = mergeState.readers.get(readerUpto);
currentValues = toMerge.get(readerUpto);
+ currentDocsWithField = docsWithField.get(readerUpto);
currentLiveDocs = currentReader.getLiveDocs();
}
docIDUpto = 0;
@@ -236,7 +247,12 @@ public abstract class DocValuesConsumer
if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
nextIsSet = true;
- currentValues.get(docIDUpto, nextValue);
+ if (currentDocsWithField.get(docIDUpto)) {
+ currentValues.get(docIDUpto, nextValue);
+ nextPointer = nextValue;
+ } else {
+ nextPointer = null;
+ }
docIDUpto++;
return true;
}
@@ -272,7 +288,10 @@ public abstract class DocValuesConsumer
OpenBitSet bitset = new OpenBitSet(dv.getValueCount());
for (int i = 0; i < reader.maxDoc(); i++) {
if (liveDocs.get(i)) {
- bitset.set(dv.getOrd(i));
+ int ord = dv.getOrd(i);
+ if (ord >= 0) {
+ bitset.set(ord);
+ }
}
}
liveTerms[sub] = new BitsFilteredTermsEnum(dv.termsEnum(), bitset);
@@ -368,7 +387,7 @@ public abstract class DocValuesConsumer
if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
nextIsSet = true;
int segOrd = dvs[readerUpto].getOrd(docIDUpto);
- nextValue = (int) map.getGlobalOrd(readerUpto, segOrd);
+ nextValue = segOrd == -1 ? -1 : (int) map.getGlobalOrd(readerUpto, segOrd);
docIDUpto++;
return true;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java Tue Aug 20 22:47:30 2013
@@ -25,6 +25,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.Bits;
/** Abstract API that produces numeric, binary and
* sorted docvalues.
@@ -56,4 +57,67 @@ public abstract class DocValuesProducer
* The returned instance need not be thread-safe: it will only be
* used by a single thread. */
public abstract SortedSetDocValues getSortedSet(FieldInfo field) throws IOException;
+
+ /** Returns a {@link Bits} at the size of <code>reader.maxDoc()</code>,
+ * with turned on bits for each docid that does have a value for this field.
+ * The returned instance need not be thread-safe: it will only be
+ * used by a single thread. */
+ public abstract Bits getDocsWithField(FieldInfo field) throws IOException;
+
+ /**
+ * A simple implementation of {@link DocValuesProducer#getDocsWithField} that
+ * returns {@code true} if a document has an ordinal >= 0
+ * <p>
+ * Codecs can choose to use this (or implement it more efficiently another way), but
+ * in most cases a Bits is unnecessary anyway: users can check this as they go.
+ */
+ public static class SortedDocsWithField implements Bits {
+ final SortedDocValues in;
+ final int maxDoc;
+
+ /** Creates a {@link Bits} returning true if the document has a value */
+ public SortedDocsWithField(SortedDocValues in, int maxDoc) {
+ this.in = in;
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public boolean get(int index) {
+ return in.getOrd(index) >= 0;
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ }
+
+ /**
+ * A simple implementation of {@link DocValuesProducer#getDocsWithField} that
+ * returns {@code true} if a document has any ordinals.
+ * <p>
+ * Codecs can choose to use this (or implement it more efficiently another way), but
+ * in most cases a Bits is unnecessary anyway: users can check this as they go.
+ */
+ public static class SortedSetDocsWithField implements Bits {
+ final SortedSetDocValues in;
+ final int maxDoc;
+
+ /** Creates a {@link Bits} returning true if the document has a value */
+ public SortedSetDocsWithField(SortedSetDocValues in, int maxDoc) {
+ this.in = in;
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public boolean get(int index) {
+ in.setDocument(index);
+ return in.nextOrd() != SortedSetDocValues.NO_MORE_ORDS;
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java Tue Aug 20 22:47:30 2013
@@ -21,13 +21,13 @@ package org.apache.lucene.codecs;
* A codec that forwards all its method calls to another codec.
* <p>
* Extend this class when you need to reuse the functionality of an existing
- * codec. For example, if you want to build a codec that redefines Lucene42's
+ * codec. For example, if you want to build a codec that redefines Lucene45's
* {@link LiveDocsFormat}:
* <pre class="prettyprint">
* public final class CustomCodec extends FilterCodec {
*
* public CustomCodec() {
- * super("CustomCodec", new Lucene42Codec());
+ * super("CustomCodec", new Lucene45Codec());
* }
*
* public LiveDocsFormat liveDocsFormat() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xNormsProducer.java Tue Aug 20 22:47:30 2013
@@ -36,6 +36,7 @@ import org.apache.lucene.index.SortedSet
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
@@ -212,4 +213,9 @@ class Lucene3xNormsProducer extends DocV
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
throw new AssertionError();
}
+
+ @Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ throw new AssertionError();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40Codec.java Tue Aug 20 22:47:30 2013
@@ -27,7 +27,6 @@ import org.apache.lucene.codecs.DocValue
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42NormsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
/**
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java Tue Aug 20 22:47:30 2013
@@ -35,6 +35,7 @@ import org.apache.lucene.index.SortedSet
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
@@ -621,6 +622,11 @@ final class Lucene40DocValuesReader exte
}
@Override
+ public Bits getDocsWithField(FieldInfo field) throws IOException {
+ return new Bits.MatchAllBits(state.segmentInfo.getDocCount());
+ }
+
+ @Override
public void close() throws IOException {
dir.close();
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java Tue Aug 20 22:47:30 2013
@@ -17,7 +17,10 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FilterCodec;
@@ -32,6 +35,7 @@ import org.apache.lucene.codecs.lucene40
import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
+import org.apache.lucene.index.SegmentWriteState;
/**
* Implements the Lucene 4.2 index format, with configurable per-field postings
@@ -42,10 +46,12 @@ import org.apache.lucene.codecs.perfield
*
* @see org.apache.lucene.codecs.lucene42 package documentation for file format details.
* @lucene.experimental
+ * @deprecated Only for reading old 4.2 segments
*/
// NOTE: if we make largish changes in a minor release, easier to just make Lucene43Codec or whatever
// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
// (it writes a minor version, etc).
+@Deprecated
public class Lucene42Codec extends Codec {
private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
@@ -129,10 +135,15 @@ public class Lucene42Codec extends Codec
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene42");
- private final NormsFormat normsFormat = new Lucene42NormsFormat();
+ private final NormsFormat normsFormat = new Lucene42NormsFormat() {
+ @Override
+ public DocValuesConsumer normsConsumer(SegmentWriteState state) throws IOException {
+ throw new UnsupportedOperationException("this codec can only be used for reading");
+ }
+ };
@Override
- public final NormsFormat normsFormat() {
+ public NormsFormat normsFormat() {
return normsFormat;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java?rev=1516012&r1=1516011&r2=1516012&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java Tue Aug 20 22:47:30 2013
@@ -123,8 +123,10 @@ import org.apache.lucene.util.packed.Blo
* <ul>
* <li> Binary doc values can be at most {@link #MAX_BINARY_FIELD_LENGTH} in length.
* </ul>
+ * @deprecated Only for reading old 4.2 segments
*/
-public final class Lucene42DocValuesFormat extends DocValuesFormat {
+@Deprecated
+public class Lucene42DocValuesFormat extends DocValuesFormat {
/** Maximum length for each binary doc values field. */
public static final int MAX_BINARY_FIELD_LENGTH = (1 << 15) - 2;
@@ -154,8 +156,7 @@ public final class Lucene42DocValuesForm
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- // note: we choose DEFAULT here (its reasonably fast, and for small bpv has tiny waste)
- return new Lucene42DocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION, acceptableOverheadRatio);
+ throw new UnsupportedOperationException("this codec can only be used for reading");
}
@Override
@@ -163,8 +164,8 @@ public final class Lucene42DocValuesForm
return new Lucene42DocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
}
- private static final String DATA_CODEC = "Lucene42DocValuesData";
- private static final String DATA_EXTENSION = "dvd";
- private static final String METADATA_CODEC = "Lucene42DocValuesMetadata";
- private static final String METADATA_EXTENSION = "dvm";
+ static final String DATA_CODEC = "Lucene42DocValuesData";
+ static final String DATA_EXTENSION = "dvd";
+ static final String METADATA_CODEC = "Lucene42DocValuesMetadata";
+ static final String METADATA_EXTENSION = "dvm";
}