You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/10/26 10:29:37 UTC
svn commit: r1027401 [1/2] - in /lucene/dev/branches/docvalues/lucene:
contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
contrib/memory/src/java/org/apache/lucene/index/memory/
src/java/org/apache/lucene/index/ src/java/org/apache/luc...
Author: simonw
Date: Tue Oct 26 08:29:35 2010
New Revision: 1027401
URL: http://svn.apache.org/viewvc?rev=1027401&view=rev
Log:
First step towards LUCENE-2700: Expose DocValues via Fields
Added:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
- copied, changed from r1021636, lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Reader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Writer.java
- copied, changed from r1021636, lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Writer.java
Removed:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Reader.java
Modified:
lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/Fields.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsEnum.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/ParallelReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Cache.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Ints.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java
Modified: lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java Tue Oct 26 08:29:35 2010
@@ -31,6 +31,7 @@ import java.util.Comparator;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.*;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.BytesRef;
@@ -394,6 +395,11 @@ public class InstantiatedIndexReader ext
public TermsEnum terms() {
return new InstantiatedTermsEnum(orderedTerms, upto, currentField);
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ return null;
+ }
};
}
@@ -422,6 +428,11 @@ public class InstantiatedIndexReader ext
}
};
}
+
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ return null;
+ }
};
}
Modified: lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/docvalues/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Oct 26 08:29:35 2010
@@ -48,6 +48,7 @@ import org.apache.lucene.index.TermFreqV
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorMapper;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -789,6 +790,12 @@ public class MemoryIndex implements Seri
public TermsEnum terms() {
return new MemoryTermsEnum(sortedFields[upto].getValue());
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ // TODO
+ throw new UnsupportedOperationException("not implemented");
+ }
};
}
@@ -819,6 +826,12 @@ public class MemoryIndex implements Seri
};
}
}
+
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ // TODO
+ throw new UnsupportedOperationException("not implemented");
+ }
};
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Oct 26 08:29:35 2010
@@ -36,14 +36,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.values.Reader;
-import org.apache.lucene.index.values.Values;
-import org.apache.lucene.index.values.ValuesEnum;
-import org.apache.lucene.index.values.Reader.Source;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.FloatsRef;
-import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.BytesRef;
@@ -996,264 +989,7 @@ class DirectoryReader extends IndexReade
}
return commits;
- }
-
- public Reader getIndexValues(String field) {
- ensureOpen();
- if (subReaders.length == 1) {
- return subReaders[0].getIndexValues(field);
- }
- return new MultiValueReader(field);
- }
-
- private class MultiValueReader extends Reader {
-
- private String id;
- private Values value;
-
- public MultiValueReader(String id) {
- this.id = id;
- for (SegmentReader reader : subReaders) {
- FieldInfo fieldInfo = reader.fieldInfos().fieldInfo(id);
- if(fieldInfo != null){
- value = fieldInfo.getIndexValues();
- break;
- }
- }
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new MultiValuesEnum(id, value);
- }
-
- @Override
- public Source load() throws IOException {
- return new MultiSource(id);
- }
-
- public void close() throws IOException {
- //
- }
-
- }
-
- private class MultiValuesEnum extends ValuesEnum {
- private int numDocs_ = 0;
- private int pos = -1;
- private int start = 0;
- private final String id;
- private final ValuesEnum[] enumCache;
- private ValuesEnum current;
-
- protected MultiValuesEnum(String id, Values enumType) {
- super(enumType);
- enumCache = new ValuesEnum[subReaders.length];
- this.id = id;
- }
-
- @Override
- public void close() throws IOException {
- for (ValuesEnum valuesEnum : enumCache) {
- if(valuesEnum != null)
- valuesEnum.close();
- }
- }
-
- @Override
- public int advance( int target) throws IOException {
- int n = target - start;
- do {
- if(target >= maxDoc)
- return pos = NO_MORE_DOCS;
- if (n >= numDocs_) {
- int idx = readerIndex(target);
- if (enumCache[idx] == null) {
- try {
- Reader indexValues = subReaders[idx].getIndexValues(id);
- if (indexValues != null) // nocommit does that work with default
- // values?
- enumCache[idx] = indexValues.getEnum(this.attributes());
- else
- enumCache[idx] = new DummyEnum(this.attributes(),
- subReaders[idx].maxDoc(), attr.type());
- } catch (IOException ex) {
- // nocommit what to do here?
- throw new RuntimeException(ex);
- }
- }
- current = enumCache[idx];
- start = starts[idx];
- numDocs_ = subReaders[idx].maxDoc();
- n = target - start;
- }
- target = start+numDocs_;
- } while ((n = current.advance(n)) == NO_MORE_DOCS);
- return pos = start+current.docID();
- }
-
-
- @Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- return advance(pos+1);
- }
- }
-
- private class MultiSource extends Source {
- private int numDocs_ = 0;
- private int start = 0;
- private Source current;
- private final String id;
-
- MultiSource(String id) {
- this.id = id;
- }
-
- public long ints(int docID) {
- int n = docID - start;
- if(n >= numDocs_) {
- int idx = readerIndex(docID);
- try{
- current = subReaders[idx].getIndexValuesCache().getInts(id);
- if(current == null) //nocommit does that work with default values?
- current = new DummySource();
- }catch(IOException ex) {
- // nocommit what to do here?
- throw new RuntimeException(ex);
- }
- start = starts[idx];
- numDocs_ = subReaders[idx].maxDoc();
- n = docID - start;
- }
- return current.ints(n);
- }
-
- public double floats(int docID) {
- int n = docID - start;
- if(n >= numDocs_) {
- int idx = readerIndex(docID);
- try{
- current = subReaders[idx].getIndexValuesCache().getFloats(id);
- if(current == null) //nocommit does that work with default values?
- current = new DummySource();
- }catch(IOException ex) {
- // nocommit what to do here?
- throw new RuntimeException(ex);
- }
- numDocs_ = subReaders[idx].maxDoc();
-
- start = starts[idx];
- n = docID - start;
- }
- return current.floats(n);
- }
-
- public BytesRef bytes(int docID) {
- int n = docID - start;
- if(n >= numDocs_) {
- int idx = readerIndex(docID);
- try{
- current = subReaders[idx].getIndexValuesCache().getBytes(id);
- if(current == null) //nocommit does that work with default values?
- current = new DummySource();
- }catch(IOException ex) {
- // nocommit what to do here?
- throw new RuntimeException(ex);
- }
- numDocs_ = subReaders[idx].maxDoc();
- start = starts[idx];
- n = docID - start;
- }
- return current.bytes(n);
- }
-
- public long ramBytesUsed() {
- return current.ramBytesUsed();
- }
-
- }
-
- private static class DummySource extends Source {
- private final BytesRef ref = new BytesRef();
- @Override
- public BytesRef bytes(int docID) {
- return ref;
- }
-
-
- @Override
- public double floats(int docID) {
- return 0.0d;
- }
-
- @Override
- public long ints(int docID) {
- return 0;
- }
-
- public long ramBytesUsed() {
- return 0;
- }
- }
-
- private static class DummyEnum extends ValuesEnum {
- private int pos = -1;
- private final int maxDoc;
-
- public DummyEnum(AttributeSource source, int maxDoc, Values type) {
- super(source, type);
- this.maxDoc = maxDoc;
- switch (type) {
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- // nocommit - this is not correct for Fixed_straight
- BytesRef bytes = attr.bytes();
- bytes.length = 0;
- bytes.offset = 0;
- break;
- case PACKED_INTS:
- case PACKED_INTS_FIXED:
- LongsRef ints = attr.ints();
- ints.set(0);
- break;
-
- case SIMPLE_FLOAT_4BYTE:
- case SIMPLE_FLOAT_8BYTE:
- FloatsRef floats = attr.floats();
- floats.set(0d);
- break;
- default:
- throw new IllegalArgumentException("unknown Values type: " + type);
- }
- }
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public int advance(int target) throws IOException {
- return pos = (pos < maxDoc ? target: NO_MORE_DOCS);
- }
- @Override
- public int docID() {
- return pos;
- }
- @Override
- public int nextDoc() throws IOException {
- return advance(pos+1);
- }
-
- }
-
+ }
private static final class ReaderCommit extends IndexCommit {
private String segmentsFileName;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Tue Oct 26 08:29:35 2010
@@ -18,11 +18,13 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.values.Ints;
import org.apache.lucene.index.values.Floats;
import org.apache.lucene.index.values.Bytes;
import org.apache.lucene.index.values.ValuesAttribute;
import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.values.codec.DocValuesConsumer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FloatsRef;
import org.apache.lucene.util.LongsRef;
@@ -48,154 +50,33 @@ final class DocFieldProcessor extends Do
final FieldInfos fieldInfos = new FieldInfos();
final DocFieldConsumer consumer;
final StoredFieldsWriter fieldsWriter;
- final private Map<String,IndexValuesProcessor> indexValues = new HashMap<String,IndexValuesProcessor>();
+ final private Map<String,DocValuesConsumer> docValues = new HashMap<String,DocValuesConsumer>();
+ private FieldsConsumer fieldsConsumer; // TODO this should be encapsulated in DocumentsWriter
- synchronized IndexValuesProcessor getProcessor(Directory dir, String segment, String name, ValuesAttribute attr, FieldInfo fieldInfo)
- throws IOException {
- if(attr == null)
- return null;
- IndexValuesProcessor p = indexValues.get(name);
- if (p == null) {
- org.apache.lucene.index.values.Values v = attr.type();
- final String id = segment + "_" + fieldInfo.number;
- switch(v) {
- case PACKED_INTS:
- p = new IntValuesProcessor(dir, id, false);
- break;
- case PACKED_INTS_FIXED:
- p = new IntValuesProcessor(dir, id, true);
- break;
- case SIMPLE_FLOAT_4BYTE:
- p = new FloatValuesProcessor(dir, id, 4);
- break;
- case SIMPLE_FLOAT_8BYTE:
- p = new FloatValuesProcessor(dir, id, 8);
- break;
- case BYTES_FIXED_STRAIGHT:
- p = new BytesValuesProcessor(dir, id, true, null, Bytes.Mode.STRAIGHT);
- break;
- case BYTES_FIXED_DEREF:
- p = new BytesValuesProcessor(dir, id, true, null, Bytes.Mode.DEREF);
- break;
- case BYTES_FIXED_SORTED:
- p = new BytesValuesProcessor(dir, id, true, attr.bytesComparator(), Bytes.Mode.SORTED);
- break;
- case BYTES_VAR_STRAIGHT:
- p = new BytesValuesProcessor(dir, id, false, null, Bytes.Mode.STRAIGHT);
- break;
- case BYTES_VAR_DEREF:
- p = new BytesValuesProcessor(dir, id, false, null, Bytes.Mode.DEREF);
- break;
- case BYTES_VAR_SORTED:
- p = new BytesValuesProcessor(dir, id, false, attr.bytesComparator(), Bytes.Mode.SORTED);
- break;
- }
- fieldInfo.setIndexValues(v);
- indexValues.put(name, p);
- }
-
- return p;
- }
-
- static abstract class IndexValuesProcessor {
- public abstract void add(int docID, String name, ValuesAttribute attr) throws IOException;
- public abstract void finish(int docCount) throws IOException;
- public abstract void files(Collection<String> files) throws IOException;
- }
-
- static class FloatValuesProcessor extends IndexValuesProcessor {
- private final Writer writer;
- private final String id;
-
- public FloatValuesProcessor(Directory dir, String id, int precision) throws IOException {
- this.id = id;
- writer = Floats.getWriter(dir, id, precision);
- }
-
- @Override
- public void add(int docID, String name, ValuesAttribute attr) throws IOException {
- final FloatsRef floats = attr.floats();
- if(floats != null) {
- writer.add(docID, floats.get());
- return;
- }
- throw new IllegalArgumentException("could not extract float/double from field " + name);
- }
-
- @Override
- public void finish(int docCount) throws IOException {
- writer.finish(docCount);
- }
-
- @Override
- public void files(Collection<String> files) {
- Floats.files(id, files);
- }
- }
-
- static class IntValuesProcessor extends IndexValuesProcessor {
- private final Writer writer;
- private final String id;
-
- public IntValuesProcessor(Directory dir, String id, boolean fixedArray) throws IOException {
- this.id = id;
- writer = Ints.getWriter(dir, id, fixedArray);
- }
-
- @Override
- public void add(int docID, String name, ValuesAttribute attr) throws IOException {
- final LongsRef ints = attr.ints();
- if(ints != null) {
- writer.add(docID, ints.get());
- return;
- }
- throw new IllegalArgumentException("could not extract int/long from field " + name);
- }
-
- @Override
- public void finish(int docCount) throws IOException {
- writer.finish(docCount);
- }
-
- @Override
- public void files(Collection<String> files) throws IOException {
- Ints.files(id, files);
- }
- }
-
- static class BytesValuesProcessor extends IndexValuesProcessor {
- private final Writer writer;
- private final String id;
- private final Directory dir;
-
- public BytesValuesProcessor(Directory dir, String id, boolean fixedSize, Comparator<BytesRef> comp, Bytes.Mode mode) throws IOException {
- this.id = id;
- writer = Bytes.getWriter(dir, id, mode,comp, fixedSize);
- this.dir = dir;
- }
-
- // nocommit -- make this thread private and not sync'd
- @Override
- public synchronized void add(int docID, String name, ValuesAttribute attr) throws IOException {
- final BytesRef bytes = attr.bytes();
- if(bytes != null) {
- writer.add(docID, bytes);
- return;
+ synchronized DocValuesConsumer docValuesConsumer(Directory dir,
+ String segment, String name, ValuesAttribute attr, FieldInfo fieldInfo)
+ throws IOException {
+ DocValuesConsumer valuesConsumer;
+ if ((valuesConsumer = docValues.get(name)) == null) {
+ fieldInfo.setIndexValues(attr.type());
+
+ if(fieldsConsumer == null) {
+ /* nocommit -- this is a hack and only works since DocValuesCodec supports initializing the FieldsConsumer twice.
+ * we need to find a way that allows us to obtain a FieldsConsumer per DocumentsWriter. Currently some codecs rely on
+ * the SegmentsWriteState passed in right at the moment when the segment is flushed (doccount etc) but we need the consumer earlier
+ * to support docvalues and later on stored fields too.
+ */
+ SegmentWriteState state = docWriter.segWriteState();
+ fieldsConsumer = state.codec.fieldsConsumer(state);
}
- throw new IllegalArgumentException("could not extract byte[] from field " + name);
+ valuesConsumer = fieldsConsumer.addValuesField(fieldInfo);
+ docValues.put(name, valuesConsumer);
}
+ return valuesConsumer;
- @Override
- public void finish(int docCount) throws IOException {
- writer.finish(docCount);
- }
-
- @Override
- public void files(Collection<String> files) throws IOException {
- Bytes.files(dir, id, files);
- }
}
+
public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer) {
this.docWriter = docWriter;
this.consumer = consumer;
@@ -221,13 +102,17 @@ final class DocFieldProcessor extends Do
fieldsWriter.flush(state);
consumer.flush(childThreadsAndFields, state);
- for(IndexValuesProcessor p : indexValues.values()) {
+ for(DocValuesConsumer p : docValues.values()) {
if (p != null) {
p.finish(state.numDocs);
p.files(state.flushedFiles);
}
}
- indexValues.clear();
+ docValues.clear();
+ if(fieldsConsumer != null) {
+ fieldsConsumer.close(); // nocommit this should go away
+ fieldsConsumer = null;
+ }
// Important to save after asking consumer to flush so
// consumer can alter the FieldInfo* if necessary. EG,
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Tue Oct 26 08:29:35 2010
@@ -20,14 +20,12 @@ package org.apache.lucene.index;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import java.util.Map.Entry;
import java.io.IOException;
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.values.ValuesAttribute;
+import org.apache.lucene.index.values.codec.DocValuesConsumer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.RamUsageEstimator;
@@ -255,17 +253,16 @@ final class DocFieldProcessorPerThread e
final DocFieldProcessorPerField perField = fields[i];
final Fieldable fieldable = perField.fields[0];
perField.consumer.processFields(perField.fields, perField.fieldCount);
+
if(!fieldable.hasFieldAttribute())
continue;
final AttributeSource attrSource = fieldable.getFieldAttributes();
if(!attrSource.hasAttribute(ValuesAttribute.class))
continue;
final ValuesAttribute attribute = attrSource.getAttribute(ValuesAttribute.class);
- final DocFieldProcessor.IndexValuesProcessor processor = docFieldProcessor
- .getProcessor(docState.docWriter.directory,
+ final DocValuesConsumer consumer = docFieldProcessor.docValuesConsumer(docState.docWriter.directory,
docState.docWriter.segment, fieldable.name(), attribute, perField.fieldInfo);
- if (processor != null)
- processor.add(docState.docID, fieldable.name(), attribute);
+ consumer.add(docState.docID, attribute);
}
if (docState.maxTermPrefix != null && docState.infoStream != null) {
docState.infoStream.println("WARNING: document contains at least one immense term (whose UTF8 encoding is longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Oct 26 08:29:35 2010
@@ -604,9 +604,13 @@ final class DocumentsWriter {
synchronized private void initFlushState(boolean onlyDocStore) {
initSegmentName(onlyDocStore);
- flushState = new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos,
- docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval(),
- writer.codecs);
+ flushState = segWriteState();
+ }
+
+ SegmentWriteState segWriteState() {
+ return new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos,
+ docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval(),
+ writer.codecs);
}
/** Returns the codec used to flush the last segment */
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java Tue Oct 26 08:29:35 2010
@@ -103,7 +103,7 @@ public final class FieldInfo {
}
}
- Values getIndexValues() {
+ public Values getIndexValues() {
return indexValues;
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/Fields.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/Fields.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/Fields.java Tue Oct 26 08:29:35 2010
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.index.values.DocValues;
+
/** Flex API for access to fields and terms
* @lucene.experimental */
@@ -31,6 +33,16 @@ public abstract class Fields {
/** Get the {@link Terms} for this field. This may return
* null if the field does not exist. */
public abstract Terms terms(String field) throws IOException;
+
+ /**
+ * Returns {@link DocValues} for the current field.
+ *
+ * @param field the field name
+ * @return the {@link DocValues} for this field or <code>null</code> if not
+ * applicable.
+ * @throws IOException
+ */
+ public abstract DocValues docValues(String field) throws IOException;
public final static Fields[] EMPTY_ARRAY = new Fields[0];
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsEnum.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldsEnum.java Tue Oct 26 08:29:35 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.ValuesEnum;
import org.apache.lucene.util.AttributeSource;
@@ -57,6 +58,16 @@ public abstract class FieldsEnum {
* will not return null. */
public abstract TermsEnum terms() throws IOException;
+ /**
+ * Returns {@link DocValues} for the current field.
+ *
+ * @return the {@link DocValues} for this field or <code>null</code> if not
+ * applicable.
+ * @throws IOException
+ */
+ public abstract DocValues docValues() throws IOException;
+
+
public final static FieldsEnum[] EMPTY_ARRAY = new FieldsEnum[0];
/** Provides zero fields */
@@ -71,5 +82,10 @@ public abstract class FieldsEnum {
public TermsEnum terms() {
throw new IllegalStateException("this method should never be called");
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ throw new IllegalStateException("this method should never be called");
+ }
};
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Oct 26 08:29:35 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.ValuesEnum;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
@@ -59,6 +60,11 @@ public class FilterIndexReader extends I
public Terms terms(String field) throws IOException {
return in.terms(field);
}
+
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ return in.docValues(field);
+ }
}
/** Base class for filtering {@link Terms}
@@ -117,6 +123,11 @@ public class FilterIndexReader extends I
public TermsEnum terms() throws IOException {
return in.terms();
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ return in.docValues();
+ }
}
/** Base class for filtering {@link TermsEnum} implementations. */
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java Tue Oct 26 08:29:35 2010
@@ -22,7 +22,7 @@ import org.apache.lucene.document.FieldS
import org.apache.lucene.search.Similarity;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.values.Cache;
-import org.apache.lucene.index.values.Reader;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.store.*;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -117,6 +117,9 @@ public abstract class IndexReader implem
public static final FieldOption TERMVECTOR_WITH_OFFSET = new FieldOption ("TERMVECTOR_WITH_OFFSET");
/** All fields with termvectors with offset values and position values enabled */
public static final FieldOption TERMVECTOR_WITH_POSITION_OFFSET = new FieldOption ("TERMVECTOR_WITH_POSITION_OFFSET");
+ /** All fields holding doc values */
+ public static final FieldOption DOC_VALUES = new FieldOption ("DOC_VALUES");
+
}
private boolean closed;
@@ -1374,10 +1377,13 @@ public abstract class IndexReader implem
public int getTermInfosIndexDivisor() {
throw new UnsupportedOperationException("This reader does not support this method.");
}
-
- // nocommit -- should this expose the iterator API via Fields and access Source only via getIndexValuesCache?
- public Reader getIndexValues(String field) {
- throw new UnsupportedOperationException();
+
+ public DocValues docValues(String field) throws IOException {
+ final Fields fields = fields();
+ if (fields == null) {
+ return null;
+ }
+ return fields.docValues(field);
}
private final Cache indexValuesCache = new Cache(this);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java Tue Oct 26 08:29:35 2010
@@ -22,6 +22,10 @@ import java.util.Map;
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
+
+import org.apache.lucene.index.values.DocValues;
+import org.apache.lucene.index.values.MultiDocValues;
+import org.apache.lucene.index.values.MultiDocValues.DocValuesIndex;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.ReaderUtil.Gather; // for javadocs
import org.apache.lucene.util.Bits;
@@ -46,6 +50,7 @@ public final class MultiFields extends F
private final Fields[] subs;
private final ReaderUtil.Slice[] subSlices;
private final Map<String,Terms> terms = new HashMap<String,Terms>();
+ private final Map<String,DocValues> docValues = new HashMap<String,DocValues>();
/** Returns a single {@link Fields} instance for this
* reader, merging fields/terms/docs/positions on the
@@ -186,6 +191,12 @@ public final class MultiFields extends F
return fields.terms(field);
}
}
+
+ /** This method may return null if the field does not exist.*/
+ public static DocValues getDocValues(IndexReader r, String field) throws IOException {
+ final Fields fields = getFields(r);
+ return fields == null? null: fields.docValues(field);
+ }
/** Returns {@link DocsEnum} for the specified field &
* term. This may return null if the term does not
@@ -270,5 +281,35 @@ public final class MultiFields extends F
return result;
}
+
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ final DocValues result;
+
+ if (!docValues.containsKey(field)) {
+
+ // Lazy init: first time this field is requested, we
+ // create & add to docValues:
+ final List<MultiDocValues.DocValuesIndex> subs2 = new ArrayList<MultiDocValues.DocValuesIndex>();
+ final List<ReaderUtil.Slice> slices2 = new ArrayList<ReaderUtil.Slice>();
+
+ // Gather all sub-readers that share this field
+ for(int i=0;i<subs.length;i++) {
+ final DocValues values = subs[i].docValues(field);
+ if (values != null) {
+ subs2.add(new MultiDocValues.DocValuesIndex(values, i));
+ slices2.add(subSlices[i]);
+ }
+ }
+ result = subs2.isEmpty()?null: new MultiDocValues(subs2.toArray(DocValuesIndex.EMPTY_ARRAY),
+ slices2.toArray(ReaderUtil.Slice.EMPTY_ARRAY));
+ docValues.put(field, result);
+ } else {
+ result = docValues.get(field);
+ }
+
+ return result; }
+
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Tue Oct 26 08:29:35 2010
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.index.values.DocValues;
+import org.apache.lucene.index.values.MultiDocValues;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ReaderUtil;
@@ -42,6 +44,8 @@ public final class MultiFieldsEnum exte
// Re-used TermsEnum
private final MultiTermsEnum terms;
+ private final MultiDocValues docValues;
+
private String currentField;
@@ -50,6 +54,7 @@ public final class MultiFieldsEnum exte
public MultiFieldsEnum(FieldsEnum[] subs, ReaderUtil.Slice[] subSlices) throws IOException {
terms = new MultiTermsEnum(subSlices);
queue = new FieldMergeQueue(subs.length);
+ docValues = new MultiDocValues(subSlices);
top = new FieldsEnumWithSlice[subs.length];
// Init q
@@ -138,5 +143,19 @@ public final class MultiFieldsEnum exte
return fieldsA.current.compareTo(fieldsB.current) < 0;
}
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ final List<MultiDocValues.DocValuesIndex> values = new ArrayList<MultiDocValues.DocValuesIndex>();
+ for (int i = 0; i < numTop; i++) {
+ final DocValues docValues = top[i].fields.docValues();
+ if (docValues != null) {
+ values.add(new MultiDocValues.DocValuesIndex(docValues,
+ top[i].index));
+ }
+ }
+ // TODO return an empty docvalues instance if values are empty
+ return docValues.reset(values.toArray(MultiDocValues.DocValuesIndex.EMPTY_ARRAY));
+ }
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/ParallelReader.java Tue Oct 26 08:29:35 2010
@@ -21,7 +21,9 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.Pair;
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
import org.apache.lucene.util.BytesRef;
@@ -174,14 +176,22 @@ public class ParallelReader extends Inde
return TermsEnum.EMPTY;
}
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ assert currentReader != null;
+ return MultiFields.getDocValues(currentReader, currentField);
+ }
}
// Single instance of this, per ParallelReader instance
private class ParallelFields extends Fields {
- final HashMap<String,Terms> fields = new HashMap<String,Terms>();
+ final HashMap<String,Pair<Terms, DocValues>> fields = new HashMap<String,Pair<Terms, DocValues>>();
public void addField(String field, IndexReader r) throws IOException {
- fields.put(field, MultiFields.getFields(r).terms(field));
+ Fields multiFields = MultiFields.getFields(r);
+ fields.put(field, new Pair<Terms, DocValues>( multiFields.terms(field),
+ multiFields.docValues(field)));
}
@Override
@@ -190,11 +200,16 @@ public class ParallelReader extends Inde
}
@Override
public Terms terms(String field) throws IOException {
- return fields.get(field);
+ return fields.get(field).cur;
}
- }
- @Override
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ return fields.get(field).cud;
+ }
+ }
+
+ @Override
public Bits getDeletedDocs() {
return MultiFields.getDeletedDocs(readers.get(0));
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Tue Oct 26 08:29:35 2010
@@ -478,12 +478,7 @@ public final class SegmentInfo {
if (delFileName != null && (delGen >= YES || dir.fileExists(delFileName))) {
fileSet.add(delFileName);
}
- //nocommit - is there a better way to get all the dat / idx files?
- for(String file : dir.listAll()) {
- if(file.startsWith(name) && (file.endsWith("dat") || file.endsWith("idx"))){
- fileSet.add(file);
- }
- }
+
if (normGen != null) {
for (int i = 0; i < normGen.length; i++) {
long gen = normGen[i];
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Tue Oct 26 08:29:35 2010
@@ -33,7 +33,7 @@ import org.apache.lucene.index.codecs.Me
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.values.Bytes;
import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.Reader;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.Floats;
import org.apache.lucene.index.values.Values;
import org.apache.lucene.index.values.Writer;
@@ -162,9 +162,6 @@ final class SegmentMerger {
if (mergeDocStores && fieldInfos.hasVectors())
mergeVectors();
-
- mergeIndexValues();
-
return mergedDocs;
}
@@ -178,12 +175,6 @@ final class SegmentMerger {
reader.close();
}
}
-
- private void addIfExists(Set<String> files, String file, Directory dir) throws IOException{
- if(dir.fileExists(file)){
- files.add(file);
- }
- }
final List<String> createCompoundFile(String fileName, final SegmentInfo info)
throws IOException {
@@ -203,14 +194,6 @@ final class SegmentMerger {
final int numFIs = fieldInfos.size();
for (int i = 0; i < numFIs; i++) {
final FieldInfo fi = fieldInfos.fieldInfo(i);
- // Index Values aka. CSF
- if (fi.indexValues != null) {
- addIfExists(fileSet, IndexFileNames.segmentFileName(segment, Integer
- .toString(fi.number), IndexFileNames.CSF_DATA_EXTENSION), directory);
- addIfExists(fileSet, IndexFileNames.segmentFileName(segment, Integer
- .toString(fi.number), IndexFileNames.CSF_INDEX_EXTENSION),
- directory);
- }
if (fi.isIndexed && !fi.omitNorms) {
fileSet.add(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION));
break;
@@ -318,7 +301,7 @@ final class SegmentMerger {
if (mergedIndexValues == null) {
merged.setIndexValues(fiIndexValues);
} else if (mergedIndexValues != fiIndexValues) {
- // nocommit -- what to do?
+ // TODO -- can we recover from this?
throw new IllegalStateException("cannot merge field " + fi.name + " indexValues changed from " + mergedIndexValues + " to " + fiIndexValues);
}
}
@@ -331,8 +314,7 @@ final class SegmentMerger {
addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, false);
addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.INDEXED), false, false, false, false, false);
fieldInfos.add(reader.getFieldNames(FieldOption.UNINDEXED), false);
-
- // nocommit -- how should we handle index values here?
+ fieldInfos.add(reader.getFieldNames(FieldOption.DOC_VALUES), false);
}
}
fieldInfos.write(directory, segment + ".fnm");
@@ -393,77 +375,6 @@ final class SegmentMerger {
return docCount;
}
- private void mergeIndexValues() throws IOException {
- final int numFields = fieldInfos.size();
- for (int i = 0; i < numFields; i++) {
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
- final Values v = fieldInfo.indexValues;
- // nocommit we need some kind of compatibility notation for values such
- // that two slighly different segments can be merged eg. fixed vs.
- // variable byte len or float32 vs. float64
-
- if (v != null) {
- int docBase = 0;
- final List<Writer.MergeState> mergeStates = new ArrayList<Writer.MergeState>();
- for (IndexReader reader : readers) {
- Reader r = reader.getIndexValues(fieldInfo.name);
- if (r != null) {
- mergeStates.add(new Writer.MergeState(r, docBase, reader
- .maxDoc(), reader.getDeletedDocs()));
- }
- docBase += reader.numDocs();
- }
- if (mergeStates.isEmpty()) {
- continue;
- }
- final String id = segment + "_" + fieldInfo.number;
- final Writer writer;
- switch (v) {
- case PACKED_INTS:
- case PACKED_INTS_FIXED:
- writer = Ints.getWriter(directory, id, true);
- break;
- case SIMPLE_FLOAT_4BYTE:
- writer = Floats.getWriter(directory, id, 4);
- break;
- case SIMPLE_FLOAT_8BYTE:
- writer = Floats.getWriter(directory, id, 8);
- break;
- case BYTES_FIXED_STRAIGHT:
- writer = Bytes.getWriter(directory, id,
- Bytes.Mode.STRAIGHT, null, true);
- break;
- case BYTES_FIXED_DEREF:
- writer = Bytes.getWriter(directory, id,
- Bytes.Mode.DEREF, null, true);
- break;
- case BYTES_FIXED_SORTED:
- // nocommit -- enable setting Comparator
- writer = Bytes.getWriter(directory, id,
- Bytes.Mode.SORTED, null, true);
- break;
- case BYTES_VAR_STRAIGHT:
- writer = Bytes.getWriter(directory, id,
- Bytes.Mode.STRAIGHT, null, false);
- break;
- case BYTES_VAR_DEREF:
- writer = Bytes.getWriter(directory, id,
- Bytes.Mode.DEREF, null, false);
- break;
- case BYTES_VAR_SORTED:
- // nocommit -- enable setting Comparator
- writer = Bytes.getWriter(directory, id,
- Bytes.Mode.SORTED, null, false);
- break;
- default:
- continue;
- }
- writer.add(mergeStates);
- writer.finish(mergedDocs);
- }
- }
- }
-
private int copyFieldsWithDeletions(final FieldsWriter fieldsWriter, final IndexReader reader,
final FieldsReader matchingFieldsReader)
throws IOException, MergeAbortedException, CorruptIndexException {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java Tue Oct 26 08:29:35 2010
@@ -44,7 +44,7 @@ import org.apache.lucene.index.codecs.Co
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.values.Bytes;
import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.Reader;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.Floats;
import org.apache.lucene.index.values.Values;
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
@@ -141,7 +141,6 @@ public class SegmentReader extends Index
// Ask codec for its Fields
fields = si.getCodec().fieldsProducer(new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor));
assert fields != null;
- openIndexValuesReaders(cfsDir, si);
success = true;
} finally {
if (!success) {
@@ -155,57 +154,8 @@ public class SegmentReader extends Index
// not assigned yet).
this.origInstance = origInstance;
}
-
- final Map<String,Reader> indexValues = new HashMap<String,Reader>();
-
- // Only opens files... doesn't actually load any values
- private void openIndexValuesReaders(Directory dir, SegmentInfo si) throws IOException {
- final int numFields = fieldInfos.size();
- for(int i=0;i<numFields;i++) {
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
- final Values v = fieldInfo.getIndexValues();
- final String field = fieldInfo.name;
- final String id = IndexFileNames.segmentFileName(segment, Integer
- .toString(fieldInfo.number), "");
- // nocommit - externalize the filenames
- if (v != null && dir.fileExists(id+".dat")) {
- switch(v) {
- case PACKED_INTS:
- indexValues.put(field, Ints.getReader(dir, id, false));
- break;
- case PACKED_INTS_FIXED:
- indexValues.put(field, Ints.getReader(dir, id, true));
- break;
- case SIMPLE_FLOAT_4BYTE:
- indexValues.put(field, Floats.getReader(dir, id, si.docCount));
- break;
- case SIMPLE_FLOAT_8BYTE:
- indexValues.put(field, Floats.getReader(dir, id, si.docCount));
- break;
- case BYTES_FIXED_STRAIGHT:
- indexValues.put(field, Bytes.getReader(dir, id, Bytes.Mode.STRAIGHT, true, si.docCount));
- break;
- case BYTES_FIXED_DEREF:
- indexValues.put(field, Bytes.getReader(dir, id, Bytes.Mode.DEREF, true, si.docCount));
- break;
- case BYTES_FIXED_SORTED:
- indexValues.put(field, Bytes.getReader(dir, id, Bytes.Mode.SORTED, true, si.docCount));
- break;
- case BYTES_VAR_STRAIGHT:
- indexValues.put(field, Bytes.getReader(dir, id, Bytes.Mode.STRAIGHT, false, si.docCount));
- break;
- case BYTES_VAR_DEREF:
- indexValues.put(field, Bytes.getReader(dir, id, Bytes.Mode.DEREF, false, si.docCount));
- break;
- case BYTES_VAR_SORTED:
- indexValues.put(field, Bytes.getReader(dir, id, Bytes.Mode.SORTED, false, si.docCount));
- break;
- default:
- throw new IllegalStateException("unrecognized index values mode " + v);
- }
- }
- }
- }
+
+
synchronized TermVectorsReader getTermVectorsReaderOrig() {
return termVectorsReaderOrig;
@@ -253,17 +203,9 @@ public class SegmentReader extends Index
if (origInstance != null) {
FieldCache.DEFAULT.purge(origInstance);
}
- closeIndexValuesReaders();
}
}
- private void closeIndexValuesReaders() throws IOException {
- for (Reader reader : indexValues.values()) {
- reader.close();
- }
- }
-
-
synchronized void openDocStores(SegmentInfo si) throws IOException {
assert si.name.equals(segment);
@@ -1026,6 +968,9 @@ public class SegmentReader extends Index
fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) {
fieldSet.add(fi.name);
}
+ else if (fi.indexValues != null && fieldOption == IndexReader.FieldOption.DOC_VALUES) {
+ fieldSet.add(fi.name);
+ }
}
return fieldSet;
}
@@ -1344,9 +1289,9 @@ public class SegmentReader extends Index
public int getTermInfosIndexDivisor() {
return core.termsIndexDivisor;
}
-
+
@Override
- public Reader getIndexValues(String field) {
- return core.indexValues.get(field);
+ public DocValues docValues(String field) throws IOException {
+ return core.fields.docValues(field);
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Tue Oct 26 08:29:35 2010
@@ -20,9 +20,17 @@ package org.apache.lucene.index.codecs;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.values.DocValues;
+import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.values.Values;
+import org.apache.lucene.index.values.codec.DocValuesConsumer;
import java.io.IOException;
import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.List;
/** Abstract API that consumes terms, doc, freq, prox and
* payloads postings. Concrete implementations of this
@@ -35,6 +43,13 @@ public abstract class FieldsConsumer imp
/** Add a new field */
public abstract TermsConsumer addField(FieldInfo field) throws IOException;
+
+ /** Adds a new DocValuesField */
+ public /*abstract*/ DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
+ throw new UnsupportedOperationException("docvalues are not supported");
+ }
+
+
/** Called when we are done adding everything. */
public abstract void close() throws IOException;
@@ -45,8 +60,39 @@ public abstract class FieldsConsumer imp
String field;
while((field = fieldsEnum.next()) != null) {
mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
- final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
- termsConsumer.merge(mergeState, fieldsEnum.terms());
+ assert mergeState.fieldInfo != null : "FieldInfo for field is null: "+ field;
+ TermsEnum terms = fieldsEnum.terms();
+ if(terms != null) {
+ final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
+ termsConsumer.merge(mergeState, terms);
+ }
+
+ DocValues docValues = fieldsEnum.docValues(); // fix this - does not work due to multi fields
+ if(docValues != null) {
+ // TODO we need some kind of compatibility notation for values such
+ // that two slighly different segments can be merged eg. fixed vs.
+ // variable byte len or float32 vs. float64
+ int docBase = 0;
+ final List<Writer.MergeState> mergeStates = new ArrayList<Writer.MergeState>();
+ for (IndexReader reader : mergeState.readers) {
+ DocValues r = reader.docValues(mergeState.fieldInfo.name);
+ if (r != null) {
+ mergeStates.add(new Writer.MergeState(r, docBase, reader
+ .maxDoc(), reader.getDeletedDocs()));
+ }
+ docBase += reader.numDocs();
+ }
+ if (mergeStates.isEmpty()) {
+ continue;
+ }
+ final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo);
+ docValuesConsumer.merge(mergeStates);
+ docValuesConsumer.finish(mergeState.mergedDocCount);
+ }
+
+ // merge doc values
+//
}
}
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java Tue Oct 26 08:29:35 2010
@@ -17,10 +17,13 @@ package org.apache.lucene.index.codecs;
* limitations under the License.
*/
-import org.apache.lucene.index.Fields;
-
-import java.io.IOException;
import java.io.Closeable;
+import java.io.IOException;
+
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.values.DocValues;
/** Abstract API that consumes terms, doc, freq, prox and
* payloads postings. Concrete implementations of this
@@ -33,4 +36,33 @@ import java.io.Closeable;
public abstract class FieldsProducer extends Fields implements Closeable {
public abstract void close() throws IOException;
public abstract void loadTermsIndex(int indexDivisor) throws IOException;
+
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ return null;
+ }
+
+ public static final FieldsProducer EMPTY = new FieldsProducer() {
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ return null;
+ }
+
+ @Override
+ public FieldsEnum iterator() throws IOException {
+ return FieldsEnum.EMPTY;
+ }
+
+ @Override
+ public void loadTermsIndex(int indexDivisor) throws IOException {
+
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+ };
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PerFieldCodecWrapper.java Tue Oct 26 08:29:35 2010
@@ -35,6 +35,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.values.DocValues;
+import org.apache.lucene.index.values.codec.DocValuesConsumer;
import org.apache.lucene.store.Directory;
@@ -112,6 +114,18 @@ public class PerFieldCodecWrapper extend
throw err;
}
}
+
+ @Override
+ public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
+ fieldsSeen.add(field.name);
+ Codec codec = getCodec(field.name);
+ FieldsConsumer fields = codecs.get(codec);
+ if (fields == null) {
+ fields = codec.fieldsConsumer(state);
+ codecs.put(codec, fields);
+ }
+ return fields.addValuesField(field);
+ }
}
private class FieldsReader extends FieldsProducer {
@@ -164,6 +178,11 @@ public class PerFieldCodecWrapper extend
return null;
}
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ return codecs.get(getCodec(current)).docValues(current);
+ }
}
@Override
@@ -207,6 +226,14 @@ public class PerFieldCodecWrapper extend
it.next().loadTermsIndex(indexDivisor);
}
}
+
+ @Override
+ public DocValues docValues(String field) throws IOException {
+ final Codec codec = getCodec(field);
+ FieldsProducer fields = codecs.get(codec);
+ assert fields != null;
+ return fields.docValues(field);
+ }
}
public FieldsProducer fieldsProducer(SegmentReadState state)
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java Tue Oct 26 08:29:35 2010
@@ -41,6 +41,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.index.codecs.standard.StandardPostingsReader; // javadocs
+import org.apache.lucene.index.values.DocValues;
/** Handles a terms dict, but decouples all details of
* doc/freqs/positions reading to an instance of {@link
@@ -245,6 +246,12 @@ public class PrefixCodedTermsReader exte
public TermsEnum terms() throws IOException {
return current.iterator();
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
private class FieldReader extends Terms implements Closeable {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java Tue Oct 26 08:29:35 2010
@@ -94,7 +94,7 @@ public class PrefixCodedTermsWriter exte
@Override
public TermsConsumer addField(FieldInfo field) {
- assert currentField == null || currentField.name.compareTo(field.name) < 0;
+ assert currentField == null || currentField.name.compareTo(field.name) < 0 : "current field name " + (currentField == null? null: currentField.name) + " given: " +field.name;
currentField = field;
TermsIndexWriterBase.FieldWriter fieldIndexWriter = termsIndexWriter.addField(field);
TermsConsumer terms = new TermsWriter(fieldIndexWriter, field, postingsWriter);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Tue Oct 26 08:29:35 2010
@@ -37,6 +37,7 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.CompoundFileReader;
import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
@@ -222,6 +223,12 @@ public class PreFlexFields extends Field
termsEnum.reset(current);
return termsEnum;
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ //DocValues are not available on PreFlex indices
+ return null;
+ }
}
private class PreTerms extends Terms {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Tue Oct 26 08:29:35 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs.s
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.index.codecs.FieldsProducer;
+import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.Terms;
@@ -103,6 +104,12 @@ class SimpleTextFieldsReader extends Fie
public TermsEnum terms() throws IOException {
return new SimpleTextTermsEnum(in.getFilePointer(), omitTF);
}
+
+ @Override
+ public DocValues docValues() throws IOException {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
private class SimpleTextTermsEnum extends TermsEnum {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java Tue Oct 26 08:29:35 2010
@@ -25,8 +25,8 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.values.Reader.SortedSource;
-import org.apache.lucene.index.values.Reader.Source;
+import org.apache.lucene.index.values.DocValues.SortedSource;
+import org.apache.lucene.index.values.DocValues.Source;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -47,8 +47,7 @@ import org.apache.lucene.util.CodecUtil;
* NOTE: Each byte[] must be <= 32768 bytes in length
* </p>
*/
-//nocommit - add mmap version
-//nocommti - add bulk copy where possible
+//TODO - add bulk copy where possible
public final class Bytes {
// don't instantiate!
@@ -59,17 +58,7 @@ public final class Bytes {
STRAIGHT, DEREF, SORTED
};
- public static void files(Directory dir, String id, Collection<String> files)
- throws IOException {
- files.add(IndexFileNames.segmentFileName(id, "",
- IndexFileNames.CSF_DATA_EXTENSION));
- final String idxFile = IndexFileNames.segmentFileName(id, "",
- IndexFileNames.CSF_INDEX_EXTENSION);
- if (dir.fileExists(idxFile)) {
- files.add(idxFile);
- }
- }
-
+
// nocommit -- i shouldn't have to specify fixed? can
// track itself & do the write thing at write time?
public static Writer getWriter(Directory dir, String id, Mode mode,
@@ -101,7 +90,7 @@ public final class Bytes {
}
// nocommit -- I can peek @ header to determing fixed/mode?
- public static Reader getReader(Directory dir, String id, Mode mode,
+ public static DocValues getValues(Directory dir, String id, Mode mode,
boolean fixedSize, int maxDoc) throws IOException {
if (fixedSize) {
if (mode == Mode.STRAIGHT) {
@@ -172,6 +161,7 @@ public final class Bytes {
static abstract class BytesWriterBase extends Writer {
+
private final Directory dir;
private final String id;
protected IndexOutput idxOut;
@@ -239,13 +229,32 @@ public final class Bytes {
bytesRef = attr.bytes();
assert bytesRef != null;
}
+
+ @Override
+ public void add(int docID, ValuesAttribute attr) throws IOException {
+ final BytesRef ref;
+ if((ref = attr.bytes()) != null) {
+ add(docID, ref);
+ }
+ }
+
+ @Override
+ public void files(Collection<String> files) throws IOException {
+ files.add(IndexFileNames.segmentFileName(id, "",
+ IndexFileNames.CSF_DATA_EXTENSION));
+ final String idxFile = IndexFileNames.segmentFileName(id, "",
+ IndexFileNames.CSF_INDEX_EXTENSION);
+ if (dir.fileExists(idxFile)) { // TODO is this correct? could be initialized lazy
+ files.add(idxFile);
+ }
+ }
}
/**
* Opens all necessary files, but does not read any data in until you call
* {@link #load}.
*/
- static abstract class BytesReaderBase extends Reader {
+ static abstract class BytesReaderBase extends DocValues {
protected final IndexInput idxIn;
protected final IndexInput datIn;
protected final int version;
@@ -270,20 +279,15 @@ public final class Bytes {
}
protected final IndexInput cloneData() {
- assert !isClosed.get():printEx();
// is never NULL
return (IndexInput) datIn.clone();
}
protected final IndexInput cloneIndex() {
- assert !isClosed.get():printEx();
return idxIn == null ? null : (IndexInput) idxIn.clone();
}
- private final AtomicBoolean isClosed = new AtomicBoolean(false);
- Exception ex;
+
public void close() throws IOException {
- assert !isClosed.getAndSet(true);
- ex =new Exception();
if (datIn != null) {
datIn.close();
}
@@ -291,11 +295,6 @@ public final class Bytes {
idxIn.close();
}
}
-
- private String printEx() {
- ex.printStackTrace();
- return ex.getMessage();
- }
}
}
\ No newline at end of file
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Cache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Cache.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Cache.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Cache.java Tue Oct 26 08:29:35 2010
@@ -23,8 +23,8 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.values.Reader.SortedSource;
-import org.apache.lucene.index.values.Reader.Source;
+import org.apache.lucene.index.values.DocValues.SortedSource;
+import org.apache.lucene.index.values.DocValues.Source;
import org.apache.lucene.util.BytesRef;
public class Cache {
@@ -42,7 +42,7 @@ public class Cache {
synchronized public Source getInts(String id) throws IOException {
Source s = ints.get(id);
if (s == null) {
- final Reader indexValues = r.getIndexValues(id);
+ final DocValues indexValues = r.docValues(id);
if (indexValues == null) {
return null;
}
@@ -56,7 +56,7 @@ public class Cache {
synchronized public Source getFloats(String id) throws IOException {
Source s = floats.get(id);
if (s == null) {
- final Reader indexValues = r.getIndexValues(id);
+ final DocValues indexValues = r.docValues(id);
if (indexValues == null) {
return null;
}
@@ -71,7 +71,7 @@ public class Cache {
Comparator<BytesRef> comp) throws IOException {
SortedSource s = sortedBytes.get(id);
if (s == null) {
- final Reader indexValues = r.getIndexValues(id);
+ final DocValues indexValues = r.docValues(id);
if (indexValues == null) {
return null;
}
@@ -87,7 +87,7 @@ public class Cache {
synchronized public Source getBytes(String id) throws IOException {
Source s = bytes.get(id);
if (s == null) {
- final Reader indexValues = r.getIndexValues(id);
+ final DocValues indexValues = r.docValues(id);
if (indexValues == null) {
return null;
}
Copied: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java (from r1021636, lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Reader.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java?p2=lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java&p1=lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Reader.java&r1=1021636&r2=1027401&rev=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Reader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java Tue Oct 26 08:29:35 2010
@@ -23,9 +23,11 @@ import java.util.Comparator;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
-public abstract class Reader implements Closeable {
+public abstract class DocValues implements Closeable {
+ public static final DocValues[] EMPTY_ARRAY = new DocValues[0];
+
public ValuesEnum getEnum() throws IOException{
return getEnum(null);
}
@@ -38,6 +40,8 @@ public abstract class Reader implements
throw new UnsupportedOperationException();
}
+ public abstract Values type();
+
/**
* Source of integer (returned as java long), per document. The underlying
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Tue Oct 26 08:29:35 2010
@@ -257,6 +257,11 @@ class FixedDerefBytesImpl {
}
}
+
+ @Override
+ public Values type() {
+ return Values.BYTES_FIXED_DEREF;
+ }
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java Tue Oct 26 08:29:35 2010
@@ -156,7 +156,7 @@ class FixedSortedBytesImpl {
}
@Override
- public org.apache.lucene.index.values.Reader.Source load() throws IOException {
+ public org.apache.lucene.index.values.DocValues.Source load() throws IOException {
return loadSorted(null);
}
@@ -254,5 +254,10 @@ class FixedSortedBytesImpl {
// do unsorted
return new DerefBytesEnum(source, cloneData(), cloneIndex(), CODEC_NAME, size);
}
+
+ @Override
+ public Values type() {
+ return Values.BYTES_FIXED_SORTED;
+ }
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Tue Oct 26 08:29:35 2010
@@ -217,5 +217,10 @@ class FixedStraightBytesImpl {
return advance(pos+1);
}
}
+
+ @Override
+ public Values type() {
+ return Values.BYTES_FIXED_STRAIGHT;
+ }
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java Tue Oct 26 08:29:35 2010
@@ -28,10 +28,6 @@ public class Floats {
private static final int INT_ZERO = Float.floatToRawIntBits(0.0f);
private static final long LONG_ZERO = Double.doubleToRawLongBits(0.0);
- public static void files(String id, Collection<String> files) {
- files.add(id + "." + IndexFileNames.CSF_DATA_EXTENSION);
- }
-
public static Writer getWriter(Directory dir, String id, int precisionBytes)
throws IOException {
if (precisionBytes != 4 && precisionBytes != 8) {
@@ -45,12 +41,14 @@ public class Floats {
}
}
- public static Reader getReader(Directory dir, String id, int maxDoc)
+ public static DocValues getValues(Directory dir, String id, int maxDoc)
throws IOException {
return new FloatsReader(dir, id, maxDoc);
}
abstract static class FloatsWriter extends Writer {
+
+
private final Directory dir;
private final String id;
private FloatsRef floatsRef;
@@ -81,6 +79,13 @@ public class Floats {
protected void add(int docID) throws IOException {
add(docID, floatsRef.get());
}
+
+ @Override
+ public void add(int docID, ValuesAttribute attr) throws IOException {
+ final FloatsRef ref;
+ if((ref = attr.floats()) != null)
+ add(docID, ref.get());
+ }
@Override
protected void setNextAttribute(ValuesAttribute attr) {
@@ -109,6 +114,13 @@ public class Floats {
} else
super.merge(state);
}
+
+ @Override
+ public void files(Collection<String> files) throws IOException {
+ files.add(IndexFileNames.segmentFileName(id, "",
+ IndexFileNames.CSF_DATA_EXTENSION));
+ }
+
}
@@ -203,7 +215,7 @@ public class Floats {
* Opens all necessary files, but does not read any data in until you call
* {@link #load}.
*/
- static class FloatsReader extends Reader {
+ static class FloatsReader extends DocValues {
private final IndexInput datIn;
private final int precisionBytes;
@@ -303,6 +315,12 @@ public class Floats {
return precisionBytes == 4 ? new Floats4Enum(source, indexInput, maxDoc)
: new Floats8EnumImpl(source, indexInput, maxDoc);
}
+
+ @Override
+ public Values type() {
+ return precisionBytes == 4 ? Values.SIMPLE_FLOAT_4BYTE
+ : Values.SIMPLE_FLOAT_8BYTE;
+ }
}
static final class Floats4Enum extends FloatsEnumImpl {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Ints.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Ints.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Ints.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Ints.java Tue Oct 26 08:29:35 2010
@@ -1,32 +1,24 @@
package org.apache.lucene.index.values;
import java.io.IOException;
-import java.util.Collection;
-import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.values.PackedIntsImpl.IntsReader;
import org.apache.lucene.index.values.PackedIntsImpl.IntsWriter;
import org.apache.lucene.store.Directory;
-//nocommit - add mmap version
-//nocommti - add bulk copy where possible
+//TODO - add bulk copy where possible
public class Ints {
private Ints() {
}
- public static void files(String id, Collection<String> files)
- throws IOException {
- files.add(IndexFileNames.segmentFileName(id, "",
- IndexFileNames.CSF_DATA_EXTENSION));
- }
public static Writer getWriter(Directory dir, String id, boolean useFixedArray)
throws IOException {
- //nocommit - implement fixed?!
+ //TODO - implement fixed?!
return new IntsWriter(dir, id);
}
- public static Reader getReader(Directory dir, String id, boolean useFixedArray) throws IOException {
+ public static DocValues getValues(Directory dir, String id, boolean useFixedArray) throws IOException {
return new IntsReader(dir, id);
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java Tue Oct 26 08:29:35 2010
@@ -17,6 +17,7 @@ package org.apache.lucene.index.values;
* limitations under the License.
*/
import java.io.IOException;
+import java.util.Collection;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.Directory;
@@ -38,6 +39,8 @@ class PackedIntsImpl {
static final int VERSION_CURRENT = VERSION_START;
static class IntsWriter extends Writer {
+
+
// nocommit - can we bulkcopy this on a merge?
private LongsRef intsRef;
private long[] docToValue;
@@ -125,13 +128,27 @@ class PackedIntsImpl {
protected void setNextAttribute(ValuesAttribute attr) {
intsRef = attr.ints();
}
+
+ @Override
+ public void add(int docID, ValuesAttribute attr) throws IOException {
+ final LongsRef ref;
+ if((ref = attr.ints()) != null) {
+ add(docID, ref.get());
+ }
+ }
+
+ @Override
+ public void files(Collection<String> files) throws IOException {
+ files.add(IndexFileNames.segmentFileName(id, "",
+ IndexFileNames.CSF_DATA_EXTENSION));
+ }
}
/**
* Opens all necessary files, but does not read any data in until you call
* {@link #load}.
*/
- static class IntsReader extends Reader {
+ static class IntsReader extends DocValues {
private final IndexInput datIn;
protected IntsReader(Directory dir, String id) throws IOException {
@@ -186,6 +203,11 @@ class PackedIntsImpl {
public ValuesEnum getEnum(AttributeSource source) throws IOException {
return new IntsEnumImpl(source, (IndexInput) datIn.clone());
}
+
+ @Override
+ public Values type() {
+ return Values.PACKED_INTS;
+ }
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1027401&r1=1027400&r2=1027401&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Tue Oct 26 08:29:35 2010
@@ -251,5 +251,10 @@ class VarDerefBytesImpl {
datIn.readBytes(ref.bytes, 0, size);
}
}
+
+ @Override
+ public Values type() {
+ return Values.BYTES_VAR_DEREF;
+ }
}
}