You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/11/19 16:21:41 UTC
svn commit: r1411240 [1/2] - in /lucene/dev/branches/lucene4547/lucene:
codecs/src/java/org/apache/lucene/codecs/bloom/
codecs/src/java/org/apache/lucene/codecs/memory/
codecs/src/java/org/apache/lucene/codecs/pulsing/
codecs/src/java/org/apache/lucene...
Author: simonw
Date: Mon Nov 19 15:21:34 2012
New Revision: 1411240
URL: http://svn.apache.org/viewvc?rev=1411240&view=rev
Log:
added on-disk / in memory impls for lucene 4.1
Added:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java
- copied, changed from r1411109, lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValues.java
- copied, changed from r1411109, lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
Removed:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SimpleDocValuesFormat.java
Modified:
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SortedDocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsBaseFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSepPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -161,7 +161,7 @@ public final class BloomFilteringPosting
IndexInput bloomIn = null;
boolean success = false;
try {
- bloomIn = state.dir.openInput(bloomFileName, state.context);
+ bloomIn = state.directory.openInput(bloomFileName, state.context);
CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, BLOOM_CODEC_VERSION,
BLOOM_CODEC_VERSION);
// // Load the hash function used in the BloomFilter
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -842,7 +842,7 @@ public final class MemoryPostingsFormat
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- final IndexInput in = state.dir.openInput(fileName, IOContext.READONCE);
+ final IndexInput in = state.directory.openInput(fileName, IOContext.READONCE);
final SortedMap<String,TermsReader> fields = new TreeMap<String,TermsReader>();
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -100,7 +100,7 @@ public abstract class PulsingPostingsFor
docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state);
pulsingReader = new PulsingPostingsReader(docsReader);
FieldsProducer ret = new BlockTreeTermsReader(
- state.dir, state.fieldInfos, state.segmentInfo,
+ state.directory, state.fieldInfos, state.segmentInfo,
pulsingReader,
state.context,
state.segmentSuffix,
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepDocValuesProducer.java Mon Nov 19 15:21:34 2012
@@ -45,7 +45,7 @@ public class SepDocValuesProducer extend
* {@link DocValues} instances for this segment and codec.
*/
public SepDocValuesProducer(SegmentReadState state) throws IOException {
- docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.getDocCount(), state.dir, state.context);
+ docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.getDocCount(), state.directory, state.context);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Mon Nov 19 15:21:34 2012
@@ -67,7 +67,7 @@ class SimpleTextFieldsReader extends Fie
final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD;
public SimpleTextFieldsReader(SegmentReadState state) throws IOException {
- in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context);
+ in = state.directory.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context);
fieldInfos = state.fieldInfos;
fields = readFields(in.clone());
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextPerDocProducer.java Mon Nov 19 15:21:34 2012
@@ -68,7 +68,7 @@ public class SimpleTextPerDocProducer ex
this.segmentSuffix = segmentSuffix;
if (anyDocValuesFields(state.fieldInfos)) {
docValues = load(state.fieldInfos, state.segmentInfo.name,
- state.segmentInfo.getDocCount(), state.dir, state.context);
+ state.segmentInfo.getDocCount(), state.directory, state.context);
} else {
docValues = new TreeMap<String, DocValues>();
}
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java Mon Nov 19 15:21:34 2012
@@ -82,7 +82,7 @@ public class SimpleTextSimpleDocValuesFo
@Override
public SimpleDVProducer fieldsProducer(SegmentReadState state) throws IOException {
- return new SimpleTextDocValuesReader(state.fieldInfos, state.dir, state.segmentInfo, state.context);
+ return new SimpleTextDocValuesReader(state.fieldInfos, state.directory, state.segmentInfo, state.context);
}
/** the .dat file contains the data.
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SortedDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SortedDocValuesConsumer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SortedDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SortedDocValuesConsumer.java Mon Nov 19 15:21:34 2012
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.DocValues.SortedSource;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.Bits;
@@ -31,6 +29,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.PriorityQueue;
+//nocommit - this needs an abort() method? to free opened files?
public abstract class SortedDocValuesConsumer {
/** This is called, in value sort order, once per unique
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesProducer.java Mon Nov 19 15:21:34 2012
@@ -53,7 +53,7 @@ public class Lucene40DocValuesProducer e
*/
public Lucene40DocValuesProducer(SegmentReadState state, String segmentSuffix) throws IOException {
if (anyDocValuesFields(state.fieldInfos)) {
- cfs = new CompoundFileDirectory(state.dir,
+ cfs = new CompoundFileDirectory(state.directory,
IndexFileNames.segmentFileName(state.segmentInfo.name,
segmentSuffix, IndexFileNames.COMPOUND_FILE_EXTENSION),
state.context, false);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsBaseFormat.java Mon Nov 19 15:21:34 2012
@@ -42,7 +42,7 @@ public final class Lucene40PostingsBaseF
@Override
public PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException {
- return new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ return new Lucene40PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -248,12 +248,12 @@ public class Lucene40PostingsFormat exte
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ PostingsReaderBase postings = new Lucene40PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
boolean success = false;
try {
FieldsProducer ret = new BlockTreeTermsReader(
- state.dir,
+ state.directory,
state.fieldInfos,
state.segmentInfo,
postings,
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java Mon Nov 19 15:21:34 2012
@@ -35,7 +35,7 @@ import org.apache.lucene.codecs.lucene40
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
import org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
-import org.apache.lucene.codecs.lucene41.values.Lucene41SimpleDocValuesFormat;
+import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
/**
@@ -120,13 +120,11 @@ public class Lucene41Codec extends Codec
return defaultFormat;
}
- private final SimpleDocValuesFormat simpleDocValuesFormat = new Lucene41SimpleDocValuesFormat();
+ private final SimpleDocValuesFormat simpleDocValuesFormat = new Lucene41DocValuesFormat();
@Override
public SimpleDocValuesFormat simpleDocValuesFormat() {
- // nocommit fixme
- //return simpleDocValuesFormat;
- return null;
+ return simpleDocValuesFormat ;
}
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsBaseFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsBaseFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsBaseFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsBaseFormat.java Mon Nov 19 15:21:34 2012
@@ -41,7 +41,7 @@ public final class Lucene41PostingsBaseF
@Override
public PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException {
- return new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ return new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -427,14 +427,14 @@ public final class Lucene41PostingsForma
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.dir,
+ PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
state.context,
state.segmentSuffix);
boolean success = false;
try {
- FieldsProducer ret = new BlockTreeTermsReader(state.dir,
+ FieldsProducer ret = new BlockTreeTermsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,
Copied: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java (from r1411109, lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesProducer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java?p2=lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java&p1=lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesProducer.java&r1=1411109&r2=1411240&rev=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java Mon Nov 19 15:21:34 2012
@@ -19,198 +19,147 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesProducer.DocValuesFactory;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.PackedInts;
-public class Lucene41BinaryDocValuesProducer extends DocValues {
-
- private final IndexInput dataIn;
- private final IndexInput indexIn;
- private final int valueLength;
- private final long dataFilePointer;
- private long bytesToRead;
-
- public Lucene41BinaryDocValuesProducer(IndexInput dataIn, IndexInput indexIn)
- throws IOException {
- this.dataIn = dataIn;
- CodecUtil.checkHeader(dataIn, Lucene41BinaryDocValuesConsumer.CODEC_NAME,
- Lucene41BinaryDocValuesConsumer.VERSION_START,
- Lucene41BinaryDocValuesConsumer.VERSION_START);
- valueLength = dataIn.readInt();
- dataFilePointer = dataIn.getFilePointer();
- CodecUtil.checkHeader(indexIn, Lucene41BinaryDocValuesConsumer.CODEC_NAME,
- Lucene41BinaryDocValuesConsumer.VERSION_START,
- Lucene41BinaryDocValuesConsumer.VERSION_START);
- bytesToRead = indexIn.readLong();
- if (valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
- this.indexIn = indexIn;
- } else {
- indexIn.close();
- this.indexIn = null;
- }
-
- }
-
- @Override
- protected Source loadSource() throws IOException {
- if (valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
- assert indexIn != null;
- return new VarStraightSource(dataIn.clone(), indexIn.clone(), bytesToRead);
- } else {
- assert indexIn == null;
- return new FixedStraightSource(dataIn.clone(), valueLength, bytesToRead);
+import static org.apache.lucene.codecs.lucene41.values.Lucene41BinaryDocValuesConsumer.*;
+
+public class Lucene41BinaryDocValues extends BinaryDocValues {
+ private final PackedInts.Reader index;
+ private final IndexInput data;
+ private final long baseOffset;
+ private final int size;
+ private int maxLength;
+
+ public Lucene41BinaryDocValues(IndexInput dataIn, long dataOffset, int size,
+ int maxLength, PackedInts.Reader index) throws IOException {
+ this.data = dataIn;
+
+ this.size = size;
+ this.maxLength = maxLength;
+ this.baseOffset = dataOffset;
+ this.index = index;
+ }
+
+ public void get(int docId, BytesRef result) {
+ try {
+ final long offset;
+ final int length;
+ if (index == null) {
+ offset = size * ((long) docId);
+ length = size;
+ } else {
+ offset = index.get(docId);
+ data.seek(baseOffset + offset);
+ // Safe to do 1+docID because we write sentinel at the end:
+ final long nextOffset = index.get(1 + docId);
+ length = (int) (nextOffset - offset);
+ }
+ result.offset = 0;
+ result.grow(length);
+ data.readBytes(result.bytes, 0, length);
+ result.length = length;
+ } catch (IOException ex) {
+ throw new IllegalStateException(
+ "failed to get value for docID: " + docId, ex);
}
}
@Override
- protected Source loadDirectSource() throws IOException {
- if (valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
- assert indexIn != null;
- return new DirectVarStraightSource(dataIn.clone(), indexIn.clone(),
- dataFilePointer);
- } else {
- assert indexIn == null;
- return new DirectFixedStraightSource(dataIn.clone(), valueLength,
- dataFilePointer);
- }
+ public int size() {
+ return size;
}
@Override
- public Type getType() {
- return valueLength == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR ? Type.BYTES_VAR_STRAIGHT
- : Type.BYTES_FIXED_STRAIGHT;
+ public boolean isFixedLength() {
+ return index == null;
}
@Override
- public void close() throws IOException {
- super.close();
- IOUtils.close(dataIn, indexIn);
- }
-
- static abstract class BytesSourceBase extends Source {
- private final PagedBytes pagedBytes;
- protected final IndexInput datIn;
- protected final IndexInput idxIn;
- protected final static int PAGED_BYTES_BITS = 15;
- protected final PagedBytes.Reader data;
- protected final long totalLengthInBytes;
-
- protected BytesSourceBase(IndexInput datIn, IndexInput idxIn,
- PagedBytes pagedBytes, long bytesToRead, Type type) throws IOException {
- super(type);
- assert bytesToRead <= datIn.length() : " file size is less than the expected size diff: "
- + (bytesToRead - datIn.length()) + " pos: " + datIn.getFilePointer();
- this.datIn = datIn;
- this.totalLengthInBytes = bytesToRead;
- this.pagedBytes = pagedBytes;
- this.pagedBytes.copy(datIn, bytesToRead);
- data = pagedBytes.freeze(true);
- this.idxIn = idxIn;
- }
+ public int maxLength() {
+ return maxLength;
}
- public final static class DirectVarStraightSource extends Source {
-
- private final PackedInts.Reader index;
- private final IndexInput data;
- private final long baseOffset;
-
- DirectVarStraightSource(IndexInput data, IndexInput index,
- long dataFilePointer) throws IOException {
- super(Type.BYTES_VAR_STRAIGHT);
- this.data = data;
- baseOffset = dataFilePointer;
- this.index = PackedInts.getDirectReader(index); // nocommit read without
- // header
- }
-
- private final int position(int docID) throws IOException {
- final long offset = index.get(docID);
- data.seek(baseOffset + offset);
- // Safe to do 1+docID because we write sentinel at the end:
- final long nextOffset = index.get(1 + docID);
- return (int) (nextOffset - offset);
- }
+ public static final class Factory extends DocValuesFactory<BinaryDocValues> {
+ private final IndexInput datIn;
+ private final IndexInput indexIn;
+ private final PackedInts.Header indexHeader;
+ private int size;
+ private int maxLength;
+ private long baseOffset;
+ private final int valueCount;
- @Override
- public BytesRef getBytes(int docID, BytesRef ref) {
+ public Factory(Directory dir,
+ SegmentInfo segmentInfo, FieldInfo field, IOContext context
+ ) throws IOException {
+ boolean success = false;
+ this.valueCount = segmentInfo.getDocCount();
+ IndexInput datIn = null;
+ IndexInput indexIn = null;
try {
- final int sizeToRead = position(docID);
- ref.offset = 0;
- ref.grow(sizeToRead);
- data.readBytes(ref.bytes, 0, sizeToRead);
- ref.length = sizeToRead;
- return ref;
- } catch (IOException ex) {
- throw new IllegalStateException("failed to get value for docID: "
- + docID, ex);
+ datIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(
+ segmentInfo, field, Lucene41DocValuesConsumer.DATA_EXTENSION),
+ context);
+ CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
+
+ this.size = datIn.readInt();
+ this.maxLength = datIn.readInt();
+ this.baseOffset = datIn.getFilePointer();
+
+ if (size == VALUE_SIZE_VAR) {
+ indexIn = dir.openInput(Lucene41DocValuesConsumer
+ .getDocValuesFileName(segmentInfo, field,
+ Lucene41DocValuesConsumer.INDEX_EXTENSION), context);
+ CodecUtil.checkHeader(indexIn, CODEC_NAME, VERSION_START,
+ VERSION_START);
+ indexHeader = PackedInts.readHeader(indexIn);
+ } else {
+ indexIn = null;
+ indexHeader = null;
+ }
+ this.indexIn = indexIn;
+ this.datIn = datIn;
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(datIn, indexIn);
+ }
}
}
- }
-
- private static final class VarStraightSource extends BytesSourceBase {
- private final PackedInts.Reader addresses;
-
- public VarStraightSource(IndexInput datIn, IndexInput idxIn,
- long bytesToRead) throws IOException {
- super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), bytesToRead,
- Type.BYTES_VAR_STRAIGHT);
- addresses = PackedInts.getReader(idxIn); // nocommit read without header
- }
-
- @Override
- public BytesRef getBytes(int docID, BytesRef bytesRef) {
- final long address = addresses.get(docID);
- return data.fillSlice(bytesRef, address,
- (int) (addresses.get(docID + 1) - address));
- }
- }
-
- private final static class FixedStraightSource extends BytesSourceBase {
- private final int size;
- public FixedStraightSource(IndexInput datIn, int size, long bytesToRead)
- throws IOException {
- super(datIn, null, new PagedBytes(PAGED_BYTES_BITS), bytesToRead,
- Type.BYTES_FIXED_STRAIGHT);
- this.size = size;
+ public BinaryDocValues getDirect() throws IOException {
+ return new Lucene41BinaryDocValues(datIn.clone(), this.baseOffset, size,
+ maxLength,
+ indexHeader == null ? null : PackedInts.getDirectReaderNoHeader(
+ indexIn.clone(), indexHeader));
+ }
+
+ public BinaryDocValues getInMemory() throws IOException {
+ // nocommit simple in memory impl
+ PackedInts.Reader indexReader = indexHeader == null ? null : PackedInts
+ .getReaderNoHeader(indexIn.clone(), indexHeader);
+ PagedBytes bytes = new PagedBytes(15);
+ bytes.copy(datIn.clone(), indexReader == null ? size * valueCount
+ : indexReader.get(indexReader.size() - 1));
+ bytes.freeze(true);
+ return new Lucene41BinaryDocValues(bytes.getDataInput(), 0, size,
+ maxLength, indexReader);
}
@Override
- public BytesRef getBytes(int docID, BytesRef bytesRef) {
- return data.fillSlice(bytesRef, size * ((long) docID), size);
- }
- }
-
- public final static class DirectFixedStraightSource extends Source {
- private final int size;
- private IndexInput data;
- private long baseOffset;
-
- DirectFixedStraightSource(IndexInput input, int size, long dataFilePointer) {
- super(Type.BYTES_FIXED_STRAIGHT);
- this.size = size;
- this.data = input;
- baseOffset = dataFilePointer;
- }
-
- @Override
- public BytesRef getBytes(int docID, BytesRef ref) {
- try {
- data.seek(baseOffset + size * ((long) docID));
- ref.offset = 0;
- ref.grow(size);
- data.readBytes(ref.bytes, 0, size);
- ref.length = size;
- return ref;
- } catch (IOException ex) {
- throw new IllegalStateException("failed to get value for docID: "
- + docID, ex);
- }
+ public void close() throws IOException {
+ IOUtils.close(datIn, indexIn);
}
}
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java Mon Nov 19 15:21:34 2012
@@ -45,14 +45,15 @@ public class Lucene41BinaryDocValuesCons
this.indexOut = indexOut;
CodecUtil.writeHeader(dataOut, CODEC_NAME, VERSION_START);
dataOut.writeInt(fixedLength ? maxLength: VALUE_SIZE_VAR);
+ dataOut.writeInt(maxLength);
CodecUtil.writeHeader(indexOut, CODEC_NAME, VERSION_START);
buffer = fixedLength ? null : new AppendingLongBuffer();
+
}
@Override
public void finish() throws IOException {
try {
- indexOut.writeLong(offset);
if (buffer != null) {
Iterator iterator = buffer.iterator();
PackedInts.Writer writer = PackedInts.getWriter(indexOut,
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java Mon Nov 19 15:21:34 2012
@@ -25,6 +25,7 @@ import org.apache.lucene.codecs.SortedDo
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -36,15 +37,18 @@ public class Lucene41DocValuesConsumer e
*/
static final String INDEX_EXTENSION = "idx";
+ static final String OFFSET_EXTENSION = "off";
+
/**
* Filename extension for data files.
*/
static final String DATA_EXTENSION = "dat";
- static final String DV_SEGMENT_SUFFIX = "dv";
+ static final String DV_SEGMENT_SUFFIX = "sdv"; // nocommit change to dv
private final SegmentInfo info;
private final Directory dir;
+ private Directory cfs;
private final IOContext context;
Lucene41DocValuesConsumer(Directory dir, SegmentInfo si, IOContext context)
@@ -54,18 +58,28 @@ public class Lucene41DocValuesConsumer e
this.context = context;
}
+ private synchronized Directory getDirectory() throws IOException {
+ if (cfs == null) {
+ cfs = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(info.name, DV_SEGMENT_SUFFIX,
+ IndexFileNames.COMPOUND_FILE_EXTENSION), context, true);
+
+ }
+ return cfs;
+ }
+
@Override
- public void close() throws IOException {}
+ public void close() throws IOException {
+ IOUtils.close(cfs);
+ }
@Override
public NumericDocValuesConsumer addNumericField(FieldInfo field,
long minValue, long maxValue) throws IOException {
- String name = IndexFileNames.segmentFileName(this.info.name + "_"
- + field.number, DV_SEGMENT_SUFFIX, DATA_EXTENSION);
+ String name = getDocValuesFileName(info, field, DATA_EXTENSION);
IndexOutput dataOut = null;
boolean success = false;
try {
- dataOut = dir.createOutput(name, context);
+ dataOut = getDirectory().createOutput(name, context);
Lucene41NumericDocValuesConsumer consumer = new Lucene41NumericDocValuesConsumer(
dataOut, minValue, maxValue, info.getDocCount());
success = true;
@@ -80,16 +94,14 @@ public class Lucene41DocValuesConsumer e
@Override
public BinaryDocValuesConsumer addBinaryField(FieldInfo field,
boolean fixedLength, int maxLength) throws IOException {
- String nameData = IndexFileNames.segmentFileName(this.info.name + "_"
- + field.number, DV_SEGMENT_SUFFIX, DATA_EXTENSION);
- String idxOut = IndexFileNames.segmentFileName(this.info.name + "_"
- + field.number, DV_SEGMENT_SUFFIX, INDEX_EXTENSION);
+ String nameData = getDocValuesFileName(info, field, DATA_EXTENSION);
+ String idxOut = getDocValuesFileName(info, field, INDEX_EXTENSION);
boolean success = false;
IndexOutput dataOut = null;
IndexOutput indexOut = null;
try {
- dataOut = dir.createOutput(nameData, context);
- indexOut = dir.createOutput(idxOut, context);
+ dataOut = getDirectory().createOutput(nameData, context);
+ indexOut = getDirectory().createOutput(idxOut, context);
Lucene41BinaryDocValuesConsumer consumer = new Lucene41BinaryDocValuesConsumer(
dataOut, indexOut, fixedLength, maxLength);
success = true;
@@ -101,11 +113,39 @@ public class Lucene41DocValuesConsumer e
}
}
+ static String getDocValuesFileName(SegmentInfo info, FieldInfo field, String extension) {
+ return IndexFileNames.segmentFileName(info.name + "_"
+ + field.number, DV_SEGMENT_SUFFIX, extension);
+ }
+
@Override
public SortedDocValuesConsumer addSortedField(FieldInfo field,
int valueCount, boolean fixedLength, int maxLength)
throws IOException {
- return null;
+ String nameData = getDocValuesFileName(info, field, DATA_EXTENSION);
+ String idxOut = getDocValuesFileName(info, field, INDEX_EXTENSION);
+ String offOut = getDocValuesFileName(info, field, OFFSET_EXTENSION);
+ boolean success = false;
+ IndexOutput dataOut = null;
+ IndexOutput indexOut = null;
+ IndexOutput offsetOut = null;
+ try {
+ dataOut = getDirectory().createOutput(nameData, context);
+ indexOut = getDirectory().createOutput(idxOut, context);
+ if (fixedLength) {
+ offsetOut = null;
+ } else {
+ offsetOut = getDirectory().createOutput(offOut, context);
+ }
+ Lucene41SortedDocValuesConsumer consumer = new Lucene41SortedDocValuesConsumer(
+ dataOut, indexOut, offsetOut, valueCount, maxLength);
+ success = true;
+ return consumer;
+ } finally {
+ if (!success) {
+ IOUtils.close(dataOut, indexOut);
+ }
+ }
}
}
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesFormat.java?rev=1411240&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesFormat.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesFormat.java Mon Nov 19 15:21:34 2012
@@ -0,0 +1,42 @@
+package org.apache.lucene.codecs.lucene41.values;
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.SimpleDVConsumer;
+import org.apache.lucene.codecs.SimpleDVProducer;
+import org.apache.lucene.codecs.SimpleDocValuesFormat;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SegmentWriteState;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class Lucene41DocValuesFormat extends SimpleDocValuesFormat {
+
+ @Override
+ public SimpleDVConsumer fieldsConsumer(SegmentWriteState state)
+ throws IOException {
+ return new Lucene41DocValuesConsumer(state.directory, state.segmentInfo, state.context);
+ }
+
+ @Override
+ public SimpleDVProducer fieldsProducer(SegmentReadState state)
+ throws IOException {
+ return new Lucene41DocValuesProducer(state.directory, state.segmentInfo, state.context);
+ }
+
+}
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java?rev=1411240&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java Mon Nov 19 15:21:34 2012
@@ -0,0 +1,114 @@
+package org.apache.lucene.codecs.lucene41.values;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.codecs.SimpleDVProducer;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.IOUtils;
+
+// nocommit
+public class Lucene41DocValuesProducer extends SimpleDVProducer {
+
+ private final CompoundFileDirectory cfs;
+ private IOContext context;
+ private final SegmentInfo info;
+ private final Map<String,DocValuesFactory<NumericDocValues>> numeric = new HashMap<String,DocValuesFactory<NumericDocValues>>();
+ private final Map<String,DocValuesFactory<BinaryDocValues>> binary = new HashMap<String,DocValuesFactory<BinaryDocValues>>();
+
+ private final Map<String,DocValuesFactory<SortedDocValues>> sorted = new HashMap<String,DocValuesFactory<SortedDocValues>>();
+
+ public Lucene41DocValuesProducer(Directory dir, SegmentInfo segmentInfo,
+ IOContext context) throws IOException {
+ this.cfs = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(
+ segmentInfo.name, Lucene41DocValuesConsumer.DV_SEGMENT_SUFFIX,
+ IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
+ this.context = context;
+ this.info = segmentInfo;
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ List<Closeable> closeables = new ArrayList<Closeable>(numeric.values());
+ closeables.addAll(binary.values());
+ closeables.addAll(sorted.values());
+ IOUtils.close(closeables);
+ } finally {
+ IOUtils.close(cfs);
+ }
+ }
+
+ @Override
+ public NumericDocValues getNumeric(FieldInfo field) throws IOException {
+ //nocommit do we need to sync that?
+ DocValuesFactory<NumericDocValues> docValuesFactory = numeric
+ .get(field.name);
+ if (docValuesFactory == null) {
+ numeric.put(field.name,
+ docValuesFactory = new Lucene41NumericDocValues.Factory(this.cfs,
+ this.info, field, context));
+ }
+ return docValuesFactory.getDirect();
+ }
+
+ @Override
+ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
+ //nocommit do we need to sync that?
+ DocValuesFactory<BinaryDocValues> docValuesFactory = binary.get(field.name);
+ if (docValuesFactory == null) {
+ binary.put(field.name,
+ docValuesFactory = new Lucene41BinaryDocValues.Factory(this.cfs,
+ this.info, field, context));
+ }
+ return docValuesFactory.getDirect();
+ }
+
+ @Override
+ public SortedDocValues getSorted(FieldInfo field) throws IOException {
+ //nocommit do we need to sync that?
+ DocValuesFactory<SortedDocValues> docValuesFactory = sorted.get(field.name);
+ if (docValuesFactory == null) {
+ sorted.put(field.name,
+ docValuesFactory = new Lucene41SortedDocValues.Factory(this.cfs,
+ this.info, field, context));
+ }
+ return docValuesFactory.getDirect();
+ }
+
+ public static abstract class DocValuesFactory<T> implements Closeable {
+
+ public abstract T getDirect() throws IOException;
+
+ public abstract T getInMemory() throws IOException;
+ }
+
+}
Copied: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValues.java (from r1411109, lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesProducer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValues.java?p2=lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValues.java&p1=lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesProducer.java&r1=1411109&r2=1411240&rev=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValues.java Mon Nov 19 15:21:34 2012
@@ -18,98 +18,90 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.DocValuesArraySource;
-import org.apache.lucene.index.DocValues;
+import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesProducer.DocValuesFactory;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedInts.Reader;
-public class Lucene41NumericDocValuesProducer extends DocValues {
+public class Lucene41NumericDocValues extends NumericDocValues {
- private final IndexInput datIn;
private final long minValue;
+ private final Reader values;
+ private final long maxValue;
- public Lucene41NumericDocValuesProducer(IndexInput input, int numDocs)
- throws IOException {
- datIn = input;
- boolean success = false;
- try {
- CodecUtil.checkHeader(datIn, Lucene41NumericDocValuesConsumer.CODEC_NAME,
- Lucene41NumericDocValuesConsumer.VERSION_START,
- Lucene41NumericDocValuesConsumer.VERSION_START);
- minValue = datIn.readLong();
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(datIn);
- }
- }
- }
-
- /**
- * Loads the actual values. You may call this more than once, eg if you
- * already previously loaded but then discarded the Source.
- */
- @Override
- protected Source loadSource() throws IOException {
- boolean success = false;
- final Source source;
- IndexInput input = null;
- try {
- input = datIn.clone();
- source = new PackedIntsSource(input, false, minValue);
- success = true;
- return source;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(input, datIn);
- }
- }
+ public Lucene41NumericDocValues(PackedInts.Reader reader, long minValue, long maxValue) {
+ this.values = reader;
+ this.minValue = minValue;
+ this.maxValue = maxValue;
}
@Override
- public void close() throws IOException {
- super.close();
- datIn.close();
+ public long get(int docID) {
+ assert docID >= 0;
+ return values.get(docID) + minValue;
}
- @Override
- public Type getType() {
- return Type.VAR_INTS;
- }
-
- @Override
- protected Source loadDirectSource() throws IOException {
- return new PackedIntsSource(datIn.clone(), true, minValue);
- }
-
- static class PackedIntsSource extends Source {
- private final PackedInts.Reader values;
+ public static final class Factory extends DocValuesFactory<NumericDocValues> {
+ private final IndexInput datIn;
+ private final PackedInts.Header header;
private final long minValue;
-
- public PackedIntsSource(IndexInput dataIn, boolean direct, long minValue)
- throws IOException {
- super(Type.VAR_INTS);
- this.minValue = minValue;
- values = direct ? PackedInts.getDirectReader(dataIn) : PackedInts
- .getReader(dataIn);
+ private final long maxValue;
+
+ public Factory(Directory dir, SegmentInfo segmentInfo, FieldInfo field, IOContext context) throws IOException {
+ this.datIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(segmentInfo, field,
+ Lucene41DocValuesConsumer.DATA_EXTENSION), context);
+ boolean success = false;
+ try {
+ CodecUtil.checkHeader(datIn, Lucene41NumericDocValuesConsumer.CODEC_NAME,
+ Lucene41NumericDocValuesConsumer.VERSION_START,
+ Lucene41NumericDocValuesConsumer.VERSION_START);
+ minValue = datIn.readLong();
+ maxValue = datIn.readLong();
+ this.header = PackedInts.readHeader(datIn);
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(datIn);
+ }
+ }
}
- @Override
- public BytesRef getBytes(int docID, BytesRef ref) {
- ref.grow(8);
- DocValuesArraySource.copyLong(ref, getInt(docID));
- return ref;
+ public NumericDocValues getDirect() throws IOException {
+ IndexInput input = datIn.clone();
+ return new Lucene41NumericDocValues(PackedInts.getDirectReaderNoHeader(input, header), minValue, maxValue);
}
+ public NumericDocValues getInMemory() throws IOException {
+ IndexInput input = datIn.clone();
+ return new Lucene41NumericDocValues(PackedInts.getReaderNoHeader(input, header), minValue, maxValue);
+ }
+
@Override
- public long getInt(int docID) {
- assert docID >= 0;
- return values.get(docID) + minValue;
+ public void close() throws IOException {
+ IOUtils.close(datIn);
}
}
-
+
+ @Override
+ public long minValue() {
+ return minValue;
+ }
+
+ @Override
+ public long maxValue() {
+ return maxValue;
+ }
+
+ @Override
+ public int size() {
+ return values.size();
+ }
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesConsumer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41NumericDocValuesConsumer.java Mon Nov 19 15:21:34 2012
@@ -22,12 +22,11 @@ public class Lucene41NumericDocValuesCon
final int bitsRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
CodecUtil.writeHeader(output, CODEC_NAME, VERSION_START);
output.writeLong(minValue);
+ output.writeLong(maxValue);
this.minValue = minValue;
- // nocommit write without header?
this.writer = PackedInts.getWriter(output, valueCount, bitsRequired, PackedInts.FASTEST);
}
-
@Override
public void add(long value) throws IOException {
writer.add(value-minValue);
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java?rev=1411240&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java Mon Nov 19 15:21:34 2012
@@ -0,0 +1,193 @@
+package org.apache.lucene.codecs.lucene41.values;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesProducer.DocValuesFactory;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.PackedInts;
+
+import static org.apache.lucene.codecs.lucene41.values.Lucene41SortedDocValuesConsumer.*;
+
+public class Lucene41SortedDocValues extends SortedDocValues {
+ private final PackedInts.Reader docToOrdIndex;
+ private final PackedInts.Reader ordToOffsetIndex;
+ private final IndexInput data;
+ private final long baseOffset;
+ private final int valueCount;
+ private int size;
+ private int maxLength;
+
+ public Lucene41SortedDocValues(IndexInput dataIn, long dataOffset, int size,
+ int maxLength, PackedInts.Reader index, PackedInts.Reader offsets)
+ throws IOException {
+ this.data = dataIn;
+ this.size = size;
+ this.maxLength = maxLength;
+ this.baseOffset = dataOffset;
+ this.docToOrdIndex = index;
+ this.valueCount = docToOrdIndex.size();
+ ordToOffsetIndex = offsets;
+ }
+
+ @Override
+ public int getOrd(int docID) {
+ return (int) docToOrdIndex.get(docID);
+ }
+
+ @Override
+ public void lookupOrd(int ord, BytesRef result) {
+ try {
+ final long offset;
+ final int length;
+ if (ordToOffsetIndex != null) {
+ offset = ordToOffsetIndex.get(ord);
+ // 1+ord is safe because we write a sentinel at the end
+ final long nextOffset = ordToOffsetIndex.get(1 + ord);
+ length = (int) (nextOffset - offset);
+ } else {
+ length = size;
+ offset = size * ord;
+ }
+ data.seek(baseOffset + offset);
+ result.offset = 0;
+ result.grow(length);
+ data.readBytes(result.bytes, 0, length);
+ result.length = length;
+ } catch (IOException ex) {
+ throw new IllegalStateException("failed", ex);
+ }
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public boolean isFixedLength() {
+ return ordToOffsetIndex == null;
+ }
+
+ @Override
+ public int maxLength() {
+ return maxLength;
+ }
+
+ public static final class Factory extends
+ DocValuesFactory<SortedDocValues> {
+ private final IndexInput datIn;
+ private final IndexInput offsetIn;
+ private final PackedInts.Header offsetHeader;
+ private final IndexInput indexIn;
+ private final PackedInts.Header indexHeader;
+ private int size;
+ private int maxLength;
+ private long baseOffset;
+
+ public Factory(Directory dir,
+ SegmentInfo segmentInfo, FieldInfo field, IOContext context)
+ throws IOException {
+ boolean success = false;
+ IndexInput datIn = null;
+ IndexInput indexIn = null;
+ IndexInput offsetIn = null;
+ try {
+ datIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(
+ segmentInfo, field, Lucene41DocValuesConsumer.DATA_EXTENSION),
+ context);
+ CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
+ indexIn = dir.openInput(Lucene41DocValuesConsumer.getDocValuesFileName(
+ segmentInfo, field, Lucene41DocValuesConsumer.INDEX_EXTENSION),
+ context);
+ CodecUtil
+ .checkHeader(indexIn, CODEC_NAME, VERSION_START, VERSION_START);
+ indexHeader = PackedInts.readHeader(indexIn);
+ this.size = datIn.readInt();
+ this.maxLength = datIn.readInt();
+ this.baseOffset = datIn.getFilePointer();
+
+ if (size == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
+ offsetIn = dir.openInput(Lucene41DocValuesConsumer
+ .getDocValuesFileName(segmentInfo, field,
+ Lucene41DocValuesConsumer.OFFSET_EXTENSION), context);
+ CodecUtil.checkHeader(offsetIn, CODEC_NAME, VERSION_START,
+ VERSION_START);
+ this.offsetHeader = PackedInts.readHeader(offsetIn);
+ } else {
+ offsetIn = null;
+ this.offsetHeader = null;
+ }
+ this.offsetIn = offsetIn;
+ this.indexIn = indexIn;
+ this.datIn = datIn;
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(datIn, indexIn, offsetIn);
+ }
+ }
+ }
+
+ public SortedDocValues getDirect() throws IOException {
+ return new Lucene41SortedDocValues(datIn.clone(), this.baseOffset, size,
+ maxLength, PackedInts.getDirectReaderNoHeader(indexIn.clone(),
+ indexHeader), offsetHeader == null ? null
+ : PackedInts.getDirectReaderNoHeader(offsetIn.clone(),
+ offsetHeader));
+ }
+
+ public Lucene41SortedDocValues getInMemory() throws IOException {
+ // nocommit simple in memory impl
+ PackedInts.Reader offsetReader = offsetHeader == null ? null : PackedInts
+ .getReaderNoHeader(offsetIn.clone(), offsetHeader);
+ PackedInts.Reader indexReader = PackedInts.getReaderNoHeader(
+ indexIn.clone(), indexHeader);
+ PagedBytes bytes = new PagedBytes(15);
+ bytes.copy(
+ datIn.clone(),
+ offsetReader == null ? size * indexReader.size() : offsetReader
+ .get(offsetReader.size() - 1));
+ bytes.freeze(true);
+ return new Lucene41SortedDocValues(bytes.getDataInput(), 0, size,
+ maxLength, indexReader, offsetReader);
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(datIn, indexIn, offsetIn);
+ }
+ }
+
+}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValuesConsumer.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValuesConsumer.java Mon Nov 19 15:21:34 2012
@@ -18,18 +18,86 @@ package org.apache.lucene.codecs.lucene4
*/
import java.io.IOException;
+import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.SortedDocValuesConsumer;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.packed.AppendingLongBuffer;
+import org.apache.lucene.util.packed.AppendingLongBuffer.Iterator;
+import org.apache.lucene.util.packed.PackedInts;
public class Lucene41SortedDocValuesConsumer extends SortedDocValuesConsumer {
+ static final int VERSION_START = -1;
+ static final String CODEC_NAME = "Lucene41Sorted";
+ static final int VALUE_SIZE_VAR = -1;
+ private final IndexOutput data;
+ private final IndexOutput index;
+ private final IndexOutput offsets;
+ private final AppendingLongBuffer offsetBuffer;
+ private final PackedInts.Writer ords;
+ private long offset = 0;
+ private final int valueCount;
+
+ public Lucene41SortedDocValuesConsumer(IndexOutput dataOut,
+ IndexOutput indexOut, IndexOutput offsetOut, int valueCount, int maxLength)
+ throws IOException {
+ int size;
+ if (offsetOut != null) {
+ size = VALUE_SIZE_VAR;
+ offsetBuffer = new AppendingLongBuffer();
+ this.offsets = offsetOut;
+ CodecUtil.writeHeader(offsetOut, CODEC_NAME, VERSION_START);
+ } else {
+ size = maxLength;
+ offsetBuffer = null;
+ this.offsets = null;
+ }
+ CodecUtil.writeHeader(dataOut, CODEC_NAME, VERSION_START);
+ dataOut.writeInt(size);
+ dataOut.writeInt(maxLength);
+ CodecUtil.writeHeader(indexOut, CODEC_NAME, VERSION_START);
+ this.data = dataOut;
+ this.index = indexOut;
+ this.valueCount = valueCount;
+ ords = PackedInts.getWriter(index, valueCount,
+ PackedInts.bitsRequired(valueCount-1), PackedInts.DEFAULT);
+ }
@Override
- public void addValue(BytesRef value) throws IOException {}
+ public void addValue(BytesRef value) throws IOException {
+ data.writeBytes(value.bytes, value.offset, value.length);
+ if (offsetBuffer != null) {
+ offsetBuffer.add(offset);
+ offset += value.length;
+ }
+ }
@Override
- public void addDoc(int ord) throws IOException {}
+ public void addDoc(int ord) throws IOException {
+ ords.add(ord);
+ }
@Override
- public void finish() throws IOException {}
+ public void finish() throws IOException {
+ try {
+ ords.finish();
+
+ if (offsetBuffer != null) {
+
+ final int bitsRequired = PackedInts.bitsRequired(offset);
+ Iterator iterator = offsetBuffer.iterator();
+ PackedInts.Writer writer = PackedInts.getWriter(offsets, valueCount+1,
+ bitsRequired, PackedInts.DEFAULT);
+ while (iterator.hasNext()) {
+ writer.add(iterator.next());
+ }
+ writer.add(offset); // total # bytes
+ writer.finish();
+ }
+ } finally {
+ IOUtils.close(data, index, offsets);
+ }
+ }
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java Mon Nov 19 15:21:34 2012
@@ -28,7 +28,7 @@ import org.apache.lucene.store.IOContext
*/
public class SegmentReadState {
/** {@link Directory} where this segment is read from. */
- public final Directory dir;
+ public final Directory directory;
/** {@link SegmentInfo} describing this segment. */
public final SegmentInfo segmentInfo;
@@ -73,7 +73,7 @@ public class SegmentReadState {
IOContext context,
int termsIndexDivisor,
String segmentSuffix) {
- this.dir = dir;
+ this.directory = dir;
this.segmentInfo = info;
this.fieldInfos = fieldInfos;
this.context = context;
@@ -84,7 +84,7 @@ public class SegmentReadState {
/** Create a {@code SegmentReadState}. */
public SegmentReadState(SegmentReadState other,
String newSegmentSuffix) {
- this.dir = other.dir;
+ this.directory = other.directory;
this.segmentInfo = other.segmentInfo;
this.fieldInfos = other.fieldInfos;
this.context = other.context;
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java Mon Nov 19 15:21:34 2012
@@ -141,6 +141,7 @@ class SortedBytesDVWriter {
}
iwBytesUsed.addAndGet(-sortedValueRamUsage);
reset();
+ consumer.finish();
}
public void abort() {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java Mon Nov 19 15:21:34 2012
@@ -219,7 +219,9 @@ final class CompoundFileWriter implement
assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written";
seenIDs.add(id);
final DirectCFSIndexOutput out;
+
if ((outputLocked = outputTaken.compareAndSet(false, true))) {
+// System.out.println(entry.file);
out = new DirectCFSIndexOutput(getOutput(), entry, false);
} else {
entry.dir = this.directory;
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/PagedBytes.java Mon Nov 19 15:21:34 2012
@@ -405,12 +405,13 @@ public final class PagedBytes {
return pointer;
}
- public final class PagedBytesDataInput extends DataInput {
+ public final class PagedBytesDataInput extends IndexInput {
private int currentBlockIndex;
private int currentBlockUpto;
private byte[] currentBlock;
PagedBytesDataInput() {
+ super("PagedBytesIndexInput");
currentBlock = blocks.get(0);
}
@@ -471,6 +472,28 @@ public final class PagedBytes {
currentBlockUpto = 0;
currentBlock = blocks.get(currentBlockIndex);
}
+
+ @Override
+ public void close() throws IOException {
+ //
+ }
+
+ @Override
+ public long getFilePointer() {
+ return currentBlockIndex * blockSize + currentBlockUpto;
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ currentBlockIndex = (int) (pos >> blockBits);
+ currentBlock = blocks.get(currentBlockIndex);
+ currentBlockUpto = (int) (pos & blockMask);
+ }
+
+ @Override
+ public long length() {
+ return upto;
+ }
}
public final class PagedBytesDataOutput extends DataOutput {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Mon Nov 19 15:21:34 2012
@@ -25,6 +25,7 @@ import org.apache.lucene.store.DataInput
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.LongsRef;
+import org.apache.lucene.util.packed.PackedInts.Format;
/**
* Simplistic compression for array of unsigned long values.
@@ -775,6 +776,11 @@ public class PackedInts {
throw new AssertionError("Unknown Writer format: " + format);
}
}
+
+ //nocommit javadoc
+ public static Reader getReaderNoHeader(DataInput in, Header header) throws IOException {
+ return getReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
+ }
/**
* Restore a {@link Reader} from a stream.
@@ -887,6 +893,11 @@ public class PackedInts {
throw new AssertionError("Unknwown format: " + format);
}
}
+
+ //nocommit javadoc
+ public static Reader getDirectReaderNoHeader(IndexInput in, Header header) throws IOException {
+ return getDirectReaderNoHeader(in, header.format, header.version, header.valueCount, header.bitsPerValue);
+ }
/**
* Construct a direct {@link Reader} from an {@link IndexInput}. This method
@@ -1117,5 +1128,31 @@ public class PackedInts {
}
}
}
+
+ //nocommit javadoc
+ public static Header readHeader(IndexInput in) throws IOException {
+ final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
+ final int bitsPerValue = in.readVInt();
+ assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
+ final int valueCount = in.readVInt();
+ final Format format = Format.byId(in.readVInt());
+ return new Header(format, valueCount, bitsPerValue, version);
+ }
+
+ public static class Header {
+
+ private final Format format;
+ private final int valueCount;
+ private final int bitsPerValue;
+ private final int version;
+
+ public Header(Format format, int valueCount, int bitsPerValue, int version) {
+ this.format = format;
+ this.valueCount = valueCount;
+ this.bitsPerValue = bitsPerValue;
+ this.version = version;
+ }
+
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java Mon Nov 19 15:21:34 2012
@@ -50,7 +50,7 @@ import org.apache.lucene.util.LuceneTest
* to this class.
*/
// nocommit don't suppress any:
-@SuppressCodecs({"Asserting", "Direct", "Memory", "Lucene41", "MockRandom", "Lucene40", "Compressing"})
+@SuppressCodecs({"Asserting", "Direct", "Memory", "MockRandom", "Lucene40", "Compressing"})
public class TestDemoDocValue extends LuceneTestCase {
public void testDemoNumber() throws IOException {
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java Mon Nov 19 15:21:34 2012
@@ -91,12 +91,12 @@ public final class Lucene41WithOrds exte
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ PostingsReaderBase postings = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
TermsIndexReaderBase indexReader;
boolean success = false;
try {
- indexReader = new FixedGapTermsIndexReader(state.dir,
+ indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@@ -112,7 +112,7 @@ public final class Lucene41WithOrds exte
success = false;
try {
FieldsProducer ret = new BlockTermsReader(indexReader,
- state.dir,
+ state.directory,
state.fieldInfos,
state.segmentInfo,
postings,
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -156,7 +156,7 @@ public final class MockFixedIntBlockPost
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new SepPostingsReader(state.dir,
+ PostingsReaderBase postingsReader = new SepPostingsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
state.context,
@@ -165,7 +165,7 @@ public final class MockFixedIntBlockPost
TermsIndexReaderBase indexReader;
boolean success = false;
try {
- indexReader = new FixedGapTermsIndexReader(state.dir,
+ indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@@ -181,7 +181,7 @@ public final class MockFixedIntBlockPost
success = false;
try {
FieldsProducer ret = new BlockTermsReader(indexReader,
- state.dir,
+ state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -180,7 +180,7 @@ public final class MockVariableIntBlockP
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new SepPostingsReader(state.dir,
+ PostingsReaderBase postingsReader = new SepPostingsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
state.context,
@@ -189,7 +189,7 @@ public final class MockVariableIntBlockP
TermsIndexReaderBase indexReader;
boolean success = false;
try {
- indexReader = new FixedGapTermsIndexReader(state.dir,
+ indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@@ -205,7 +205,7 @@ public final class MockVariableIntBlockP
success = false;
try {
FieldsProducer ret = new BlockTermsReader(indexReader,
- state.dir,
+ state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1411240&r1=1411239&r2=1411240&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Mon Nov 19 15:21:34 2012
@@ -288,7 +288,7 @@ public final class MockRandomPostingsFor
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, SEED_EXT);
- final IndexInput in = state.dir.openInput(seedFileName, state.context);
+ final IndexInput in = state.directory.openInput(seedFileName, state.context);
final long seed = in.readLong();
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " formatID=" + state.segmentSuffix + " seed=" + seed);
@@ -308,13 +308,13 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Sep postings");
}
- postingsReader = new SepPostingsReader(state.dir, state.fieldInfos, state.segmentInfo,
+ postingsReader = new SepPostingsReader(state.directory, state.fieldInfos, state.segmentInfo,
state.context, new MockIntStreamFactory(random), state.segmentSuffix);
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Standard postings");
}
- postingsReader = new Lucene41PostingsReader(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
}
if (random.nextBoolean()) {
@@ -335,7 +335,7 @@ public final class MockRandomPostingsFor
boolean success = false;
try {
- fields = new BlockTreeTermsReader(state.dir,
+ fields = new BlockTreeTermsReader(state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,
@@ -369,7 +369,7 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: fixed-gap terms index (divisor=" + state.termsIndexDivisor + ")");
}
- indexReader = new FixedGapTermsIndexReader(state.dir,
+ indexReader = new FixedGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@@ -385,7 +385,7 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: variable-gap terms index (divisor=" + state.termsIndexDivisor + ")");
}
- indexReader = new VariableGapTermsIndexReader(state.dir,
+ indexReader = new VariableGapTermsIndexReader(state.directory,
state.fieldInfos,
state.segmentInfo.name,
state.termsIndexDivisor,
@@ -405,7 +405,7 @@ public final class MockRandomPostingsFor
success = false;
try {
fields = new BlockTermsReader(indexReader,
- state.dir,
+ state.directory,
state.fieldInfos,
state.segmentInfo,
postingsReader,