You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/10/29 22:33:37 UTC
svn commit: r1028899 - in /lucene/dev/branches/docvalues/lucene/src:
java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/
java/org/apache/lucene/index/codecs/docvalues/
java/org/apache/lucene/index/values/ java/org/apache/lucene/search/ j...
Author: simonw
Date: Fri Oct 29 20:33:36 2010
New Revision: 1028899
URL: http://svn.apache.org/viewvc?rev=1028899&view=rev
Log:
LUCENE-2700: Added support for MultiFields, MultiFieldsEnum and moved actual merge code to DocValuesCodec
Removed:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Cache.java
Modified:
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java
lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Fri Oct 29 20:33:36 2010
@@ -49,7 +49,7 @@ final class DocFieldProcessor extends Do
throws IOException {
DocValuesConsumer valuesConsumer;
if ((valuesConsumer = docValues.get(name)) == null) {
- fieldInfo.setIndexValues(attr.type());
+ fieldInfo.setDocValues(attr.type());
if(fieldsConsumer == null) {
/* nocommit -- this is a hack and only works since DocValuesCodec supports initializing the FieldsConsumer twice.
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfo.java Fri Oct 29 20:33:36 2010
@@ -24,7 +24,7 @@ public final class FieldInfo {
public String name;
public boolean isIndexed;
public int number;
- Values indexValues;
+ Values docValues;
// true if term vector for this field should be stored
@@ -93,17 +93,21 @@ public final class FieldInfo {
}
}
- void setIndexValues(Values v) {
- if (indexValues != null) {
- if (indexValues != v) {
- throw new IllegalArgumentException("indexValues is already set to " + indexValues + "; cannot change to " + v);
+ void setDocValues(Values v) {
+ if (docValues != null) {
+ if (docValues != v) {
+ throw new IllegalArgumentException("indexValues is already set to " + docValues + "; cannot change to " + v);
}
} else{
- indexValues = v;
+ docValues = v;
}
}
+
+ public boolean hasDocValues() {
+ return docValues != null;
+ }
- public Values getIndexValues() {
- return indexValues;
+ public Values getDocValues() {
+ return docValues;
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/FieldInfos.java Fri Oct 29 20:33:36 2010
@@ -311,10 +311,10 @@ public final class FieldInfos {
final byte b;
- if (fi.indexValues == null) {
+ if (fi.docValues == null) {
b = 0;
} else {
- switch(fi.indexValues) {
+ switch(fi.docValues) {
case PACKED_INTS:
b = 1;
break;
@@ -346,7 +346,7 @@ public final class FieldInfos {
b = 10;
break;
default:
- throw new IllegalStateException("unhandled indexValues type " + fi.indexValues);
+ throw new IllegalStateException("unhandled indexValues type " + fi.docValues);
}
}
output.writeByte(b);
@@ -377,43 +377,41 @@ public final class FieldInfos {
boolean omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;
FieldInfo fi = addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
-
if (format <= FORMAT_INDEX_VALUES) {
final byte b = input.readByte();
-
switch(b) {
case 0:
- fi.indexValues = null;
+ fi.docValues = null;
break;
case 1:
- fi.indexValues = Values.PACKED_INTS;
+ fi.docValues = Values.PACKED_INTS;
break;
case 2:
- fi.indexValues = Values.SIMPLE_FLOAT_4BYTE;
+ fi.docValues = Values.SIMPLE_FLOAT_4BYTE;
break;
case 3:
- fi.indexValues = Values.SIMPLE_FLOAT_8BYTE;
+ fi.docValues = Values.SIMPLE_FLOAT_8BYTE;
break;
case 4:
- fi.indexValues = Values.BYTES_FIXED_STRAIGHT;
+ fi.docValues = Values.BYTES_FIXED_STRAIGHT;
break;
case 5:
- fi.indexValues = Values.BYTES_FIXED_DEREF;
+ fi.docValues = Values.BYTES_FIXED_DEREF;
break;
case 6:
- fi.indexValues = Values.BYTES_FIXED_SORTED;
+ fi.docValues = Values.BYTES_FIXED_SORTED;
break;
case 7:
- fi.indexValues = Values.BYTES_VAR_STRAIGHT;
+ fi.docValues = Values.BYTES_VAR_STRAIGHT;
break;
case 8:
- fi.indexValues = Values.BYTES_VAR_DEREF;
+ fi.docValues = Values.BYTES_VAR_DEREF;
break;
case 9:
- fi.indexValues = Values.BYTES_VAR_SORTED;
+ fi.docValues = Values.BYTES_VAR_SORTED;
break;
case 10:
- fi.indexValues = Values.PACKED_INTS_FIXED;
+ fi.docValues = Values.PACKED_INTS_FIXED;
break;
default:
throw new IllegalStateException("unhandled indexValues type " + b);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/IndexReader.java Fri Oct 29 20:33:36 2010
@@ -21,7 +21,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.values.Cache;
import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.store.*;
import org.apache.lucene.util.Bits;
@@ -1090,7 +1089,7 @@ public abstract class IndexReader implem
if (docs == null) return 0;
int n = 0;
int doc;
- while ((doc = docs.nextDoc()) != docs.NO_MORE_DOCS) {
+ while ((doc = docs.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
deleteDocument(doc);
n++;
}
@@ -1386,13 +1385,6 @@ public abstract class IndexReader implem
return fields.docValues(field);
}
- private final Cache indexValuesCache = new Cache(this);
-
- // nocommit -- don't expose readers if we have this?
- public Cache getIndexValuesCache() {
- return indexValuesCache;
- }
-
private Fields fields;
/** @lucene.internal */
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFields.java Fri Oct 29 20:33:36 2010
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.MultiDocValues;
+import org.apache.lucene.index.values.Values;
import org.apache.lucene.index.values.MultiDocValues.DocValuesIndex;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.ReaderUtil.Gather; // for javadocs
@@ -290,19 +291,30 @@ public final class MultiFields extends F
// Lazy init: first time this field is requested, we
// create & add to docValues:
- final List<MultiDocValues.DocValuesIndex> subs2 = new ArrayList<MultiDocValues.DocValuesIndex>();
- final List<ReaderUtil.Slice> slices2 = new ArrayList<ReaderUtil.Slice>();
-
+ final List<MultiDocValues.DocValuesIndex> docValuesIndex = new ArrayList<MultiDocValues.DocValuesIndex>();
+ int docsUpto = 0;
+ Values type = null;
// Gather all sub-readers that share this field
for(int i=0;i<subs.length;i++) {
- final DocValues values = subs[i].docValues(field);
+ DocValues values = subs[i].docValues(field);
+ final int start = subSlices[i].start;
+ final int length = subSlices[i].length;
if (values != null) {
- subs2.add(new MultiDocValues.DocValuesIndex(values, i));
- slices2.add(subSlices[i]);
+ if (docsUpto != start) {
+ type = values.type();
+ docValuesIndex.add(new MultiDocValues.DocValuesIndex(new MultiDocValues.DummyDocValues(start, type), docsUpto, start - docsUpto));
+ }
+ docValuesIndex.add(new MultiDocValues.DocValuesIndex(values, start, length));
+ docsUpto = start+length;
+
+ } else if (i+1 == subs.length && !docValuesIndex.isEmpty()) {
+ docValuesIndex.add(new MultiDocValues.DocValuesIndex(
+ new MultiDocValues.DummyDocValues(start, type), docsUpto, start
+ - docsUpto));
}
+
}
- result = subs2.isEmpty()?null: new MultiDocValues(subs2.toArray(DocValuesIndex.EMPTY_ARRAY),
- slices2.toArray(ReaderUtil.Slice.EMPTY_ARRAY));
+ result = docValuesIndex.isEmpty()?null: new MultiDocValues(docValuesIndex.toArray(DocValuesIndex.EMPTY_ARRAY));
docValues.put(field, result);
} else {
result = docValues.get(field);
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Fri Oct 29 20:33:36 2010
@@ -19,8 +19,10 @@ package org.apache.lucene.index;
import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.MultiDocValues;
+import org.apache.lucene.index.values.Values;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.ReaderUtil.Slice;
import java.io.IOException;
import java.util.List;
@@ -40,6 +42,8 @@ public final class MultiFieldsEnum exte
// Holds sub-readers containing field we are currently
// on, popped from queue.
private final FieldsEnumWithSlice[] top;
+ private final FieldsEnumWithSlice[] enumWithSlices;
+
private int numTop;
// Re-used TermsEnum
@@ -54,8 +58,9 @@ public final class MultiFieldsEnum exte
public MultiFieldsEnum(FieldsEnum[] subs, ReaderUtil.Slice[] subSlices) throws IOException {
terms = new MultiTermsEnum(subSlices);
queue = new FieldMergeQueue(subs.length);
- docValues = new MultiDocValues(subSlices);
+ docValues = new MultiDocValues();
top = new FieldsEnumWithSlice[subs.length];
+ List<FieldsEnumWithSlice> enumWithSlices = new ArrayList<FieldsEnumWithSlice>();
// Init q
for(int i=0;i<subs.length;i++) {
@@ -64,10 +69,13 @@ public final class MultiFieldsEnum exte
if (field != null) {
// this FieldsEnum has at least one field
final FieldsEnumWithSlice sub = new FieldsEnumWithSlice(subs[i], subSlices[i], i);
+ enumWithSlices.add(sub);
sub.current = field;
queue.add(sub);
}
}
+ this.enumWithSlices = enumWithSlices.toArray(FieldsEnumWithSlice.EMPTY_ARRAY);
+
}
@Override
@@ -119,6 +127,7 @@ public final class MultiFieldsEnum exte
}
public final static class FieldsEnumWithSlice {
+ public static final FieldsEnumWithSlice[] EMPTY_ARRAY = new FieldsEnumWithSlice[0];
final FieldsEnum fields;
final ReaderUtil.Slice slice;
final int index;
@@ -146,16 +155,37 @@ public final class MultiFieldsEnum exte
@Override
public DocValues docValues() throws IOException {
- final List<MultiDocValues.DocValuesIndex> values = new ArrayList<MultiDocValues.DocValuesIndex>();
- for (int i = 0; i < numTop; i++) {
- final DocValues docValues = top[i].fields.docValues();
- if (docValues != null) {
- values.add(new MultiDocValues.DocValuesIndex(docValues,
- top[i].index));
+ final List<MultiDocValues.DocValuesIndex> docValuesIndex = new ArrayList<MultiDocValues.DocValuesIndex>();
+ int docsUpto = 0;
+ Values type = null;
+ final int numEnums = enumWithSlices.length;
+ for (int i = 0; i < numEnums; i++) {
+ FieldsEnumWithSlice withSlice = enumWithSlices[i];
+ Slice slice = withSlice.slice;
+ final DocValues values = withSlice.fields.docValues();
+
+ final int start = slice.start;
+ final int length = slice.length;
+ if (values != null) {
+ if (docsUpto != start) {
+ type = values.type();
+ docValuesIndex.add(new MultiDocValues.DocValuesIndex(
+ new MultiDocValues.DummyDocValues(start, type), docsUpto, start
+ - docsUpto));
+ }
+ docValuesIndex.add(new MultiDocValues.DocValuesIndex(values, start,
+ length));
+ docsUpto = start + length;
+
+
+ } else if (i+1 == numEnums && !docValuesIndex.isEmpty()) {
+ docValuesIndex.add(new MultiDocValues.DocValuesIndex(
+ new MultiDocValues.DummyDocValues(start, type), docsUpto, start
+ - docsUpto));
}
}
- // TODO return an empty docvalues instance if values are empty
- return docValues.reset(values.toArray(MultiDocValues.DocValuesIndex.EMPTY_ARRAY));
+ return docValuesIndex.isEmpty() ? null : docValues.reset(docValuesIndex
+ .toArray(MultiDocValues.DocValuesIndex.EMPTY_ARRAY));
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Oct 29 20:33:36 2010
@@ -31,12 +31,7 @@ import org.apache.lucene.index.codecs.Co
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.MergeState;
import org.apache.lucene.index.codecs.FieldsConsumer;
-import org.apache.lucene.index.values.Bytes;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.DocValues;
-import org.apache.lucene.index.values.Floats;
import org.apache.lucene.index.values.Values;
-import org.apache.lucene.index.values.Writer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -296,13 +291,13 @@ final class SegmentMerger {
fi.storePositionWithTermVector, fi.storeOffsetWithTermVector,
!reader.hasNorms(fi.name), fi.storePayloads,
fi.omitTermFreqAndPositions);
- final Values fiIndexValues = fi.indexValues;
- final Values mergedIndexValues = merged.indexValues;
- if (mergedIndexValues == null) {
- merged.setIndexValues(fiIndexValues);
- } else if (mergedIndexValues != fiIndexValues) {
+ final Values fiIndexValues = fi.docValues;
+ final Values mergedDocValues = merged.docValues;
+ if (mergedDocValues == null) {
+ merged.setDocValues(fiIndexValues);
+ } else if (mergedDocValues != fiIndexValues) {
// TODO -- can we recover from this?
- throw new IllegalStateException("cannot merge field " + fi.name + " indexValues changed from " + mergedIndexValues + " to " + fiIndexValues);
+ throw new IllegalStateException("cannot merge field " + fi.name + " indexValues changed from " + mergedDocValues + " to " + fiIndexValues);
}
}
} else {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/SegmentReader.java Fri Oct 29 20:33:36 2010
@@ -968,7 +968,7 @@ public class SegmentReader extends Index
fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) {
fieldSet.add(fi.name);
}
- else if (fi.indexValues != null && fieldOption == IndexReader.FieldOption.DOC_VALUES) {
+ else if (fi.docValues != null && fieldOption == IndexReader.FieldOption.DOC_VALUES) {
fieldSet.add(fi.name);
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/FieldsConsumer.java Fri Oct 29 20:33:36 2010
@@ -65,32 +65,13 @@ public abstract class FieldsConsumer imp
final TermsConsumer termsConsumer = addField(mergeState.fieldInfo);
termsConsumer.merge(mergeState, terms);
}
-
- DocValues docValues = fieldsEnum.docValues(); // fix this - does not work due to multi fields
- if(docValues != null) {
- // TODO we need some kind of compatibility notation for values such
- // that two slighly different segments can be merged eg. fixed vs.
- // variable byte len or float32 vs. float64
- int docBase = 0;
- final List<Writer.MergeState> mergeStates = new ArrayList<Writer.MergeState>();
- for (IndexReader reader : mergeState.readers) {
- DocValues r = reader.docValues(mergeState.fieldInfo.name);
- if (r != null) {
- mergeStates.add(new Writer.MergeState(r, docBase, reader
- .maxDoc(), reader.getDeletedDocs()));
- }
- docBase += reader.numDocs();
- }
- if (mergeStates.isEmpty()) {
- continue;
- }
+ if (mergeState.fieldInfo.hasDocValues()) {
+ final DocValues docValues = fieldsEnum.docValues();
+ assert docValues != null : "DocValues are null for " + mergeState.fieldInfo.getDocValues();
final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo);
- docValuesConsumer.merge(mergeStates);
- docValuesConsumer.finish(mergeState.mergedDocCount);
+ assert docValuesConsumer != null;
+ docValuesConsumer.merge(mergeState, docValues);
}
-
- // merge doc values
-//
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesConsumer.java Fri Oct 29 20:33:36 2010
@@ -1,4 +1,5 @@
package org.apache.lucene.index.codecs.docvalues;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -18,9 +19,9 @@ package org.apache.lucene.index.codecs.d
import java.io.IOException;
import java.util.Collection;
import java.util.Comparator;
-import java.util.List;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.values.DocValues;
import org.apache.lucene.index.values.ValuesAttribute;
import org.apache.lucene.index.values.Writer;
@@ -28,22 +29,50 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+/**
+ * @lucene.experimental
+ */
+// TODO this might need to go in the codec package since is a direct relative to
+// TermsConsumer
public abstract class DocValuesConsumer {
+
public abstract void add(int docID, ValuesAttribute attr) throws IOException;
public abstract void finish(int docCount) throws IOException;
public abstract void files(Collection<String> files) throws IOException;
-
- public void merge(List<MergeState> states) throws IOException {
- for (MergeState state : states) {
- merge(state);
+
+ public void merge(org.apache.lucene.index.codecs.MergeState mergeState,
+ DocValues values) throws IOException {
+ // TODO we need some kind of compatibility notation for values such
+ // that two slightly different segments can be merged eg. fixed vs.
+ // variable byte len or float32 vs. float64
+ int docBase = 0;
+ boolean merged = false;
+ /*
+ * We ignore the given DocValues here and merge from the subReaders directly
+ * to support bulk copies on the DocValues Writer level. if this gets merged
+ * with MultiDocValues the writer can not optimize for bulk-copyable data
+ */
+ for (final IndexReader reader : mergeState.readers) {
+ final DocValues r = reader.docValues(mergeState.fieldInfo.name);
+ if (r != null) {
+ merged = true;
+ merge(new Writer.MergeState(r, docBase, reader.maxDoc(), reader
+ .getDeletedDocs()));
+ }
+ docBase += reader.numDocs();
}
+ if (merged)
+ finish(mergeState.mergedDocCount);
}
-
+
protected abstract void merge(MergeState mergeState) throws IOException;
-
-
+
+ /*
+ * specialized auxiliary MergeState is necessary since we don't want to
+ * exploit internals up to the codec ones
+ */
public static class MergeState {
public final DocValues reader;
public final int docBase;
@@ -59,9 +88,10 @@ public abstract class DocValuesConsumer
}
}
- public static DocValuesConsumer create(String segmentName, Directory directory,
- FieldInfo field, Comparator<BytesRef> comp) throws IOException {
+ public static DocValuesConsumer create(String segmentName,
+ Directory directory, FieldInfo field, Comparator<BytesRef> comp)
+ throws IOException {
final String id = segmentName + "_" + field.number;
- return Writer.create(field.getIndexValues(), id, directory, comp);
+ return Writer.create(field.getDocValues(), id, directory, comp);
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/docvalues/DocValuesProducerBase.java Fri Oct 29 20:33:36 2010
@@ -51,13 +51,13 @@ public abstract class DocValuesProducerB
final int numFields = fieldInfos.size();
for (int i = 0; i < numFields; i++) {
final FieldInfo fieldInfo = fieldInfos.fieldInfo(i);
- final Values v = fieldInfo.getIndexValues();
+ final Values v = fieldInfo.getDocValues();
final String field = fieldInfo.name;
final String id = IndexFileNames.segmentFileName(segment, Integer
- .toString(fieldInfo.number), "");
- if (v != null && dir.fileExists(id + "." + Writer.DATA_EXTENSION)) {
+ .toString(fieldInfo.number),"");
+ if (v != null && dir.fileExists(id + "." + Writer.DATA_EXTENSION)) {
docValues.put(field, loadDocValues(docCount, dir, id, v));
- }
+ }
}
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Bytes.java Fri Oct 29 20:33:36 2010
@@ -287,7 +287,9 @@ public final class Bytes {
return idxIn == null ? null : (IndexInput) idxIn.clone();
}
+ @Override
public void close() throws IOException {
+ super.close();
if (datIn != null) {
datIn.close();
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/DocValues.java Fri Oct 29 20:33:36 2010
@@ -24,24 +24,48 @@ import org.apache.lucene.util.AttributeS
import org.apache.lucene.util.BytesRef;
public abstract class DocValues implements Closeable {
-
-
+
+ private final Object lock = new Object();
+
+ private Source cachedReference;
+
public static final DocValues[] EMPTY_ARRAY = new DocValues[0];
- public ValuesEnum getEnum() throws IOException{
+ public ValuesEnum getEnum() throws IOException {
return getEnum(null);
}
- public abstract ValuesEnum getEnum(AttributeSource attrSource) throws IOException;
+ public abstract ValuesEnum getEnum(AttributeSource attrSource)
+ throws IOException;
public abstract Source load() throws IOException;
- public SortedSource loadSorted(Comparator<BytesRef> comparator) throws IOException {
+ public Source getCached(boolean load) throws IOException {
+ synchronized (lock) { // TODO make sorted source cachable too
+ if (load && cachedReference == null)
+ cachedReference = load();
+ return cachedReference;
+ }
+ }
+
+ public Source releaseCached() {
+ synchronized (lock) {
+ final Source retVal = cachedReference;
+ cachedReference = null;
+ return retVal;
+ }
+ }
+
+ public SortedSource loadSorted(Comparator<BytesRef> comparator)
+ throws IOException {
throw new UnsupportedOperationException();
}
-
+
public abstract Values type();
+ public void close() throws IOException {
+ releaseCached();
+ }
/**
* Source of integer (returned as java long), per document. The underlying
@@ -50,30 +74,34 @@ public abstract class DocValues implemen
*/
public static abstract class Source {
- public long ints(int docID) {
+ public long getInt(int docID) {
throw new UnsupportedOperationException("ints are not supported");
}
- public double floats(int docID) {
+ public double getFloat(int docID) {
throw new UnsupportedOperationException("floats are not supported");
}
- public BytesRef bytes(int docID) {
+ public BytesRef getBytes(int docID) {
throw new UnsupportedOperationException("bytes are not supported");
}
-
- /** Returns number of unique values. Some impls may
- * throw UnsupportedOperationException. */
+
+ /**
+ * Returns number of unique values. Some impls may throw
+ * UnsupportedOperationException.
+ */
public int getValueCount() {
throw new UnsupportedOperationException();
}
-
- public ValuesEnum getEnum() throws IOException{
+
+ public ValuesEnum getEnum() throws IOException {
return getEnum(null);
}
-
- // nocommit - enable obtaining enum from source since this is already in memory
- public /*abstract*/ ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
+
+ // nocommit - enable obtaining enum from source since this is already in
+ // memory
+ public/* abstract */ValuesEnum getEnum(AttributeSource attrSource)
+ throws IOException {
throw new UnsupportedOperationException();
}
@@ -83,7 +111,7 @@ public abstract class DocValues implemen
public static abstract class SortedSource extends Source {
@Override
- public BytesRef bytes(int docID) {
+ public BytesRef getBytes(int docID) {
return getByOrd(ord(docID));
}
@@ -109,5 +137,5 @@ public abstract class DocValues implemen
*/
public abstract LookupResult getByValue(BytesRef value);
}
-
+
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -157,7 +157,7 @@ class FixedDerefBytesImpl {
}
@Override
- public BytesRef bytes(int docID) {
+ public BytesRef getBytes(int docID) {
final int id = (int) index.get(docID);
if (id == 0) {
return defaultValue;
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -150,7 +150,7 @@ class FixedStraightBytesImpl {
}
@Override
- public BytesRef bytes(int docID) {
+ public BytesRef getBytes(int docID) {
bytesRef.offset = docID * size;
return bytesRef;
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Floats.java Fri Oct 29 20:33:36 2010
@@ -270,7 +270,7 @@ public class Floats {
}
@Override
- public double floats(int docID) {
+ public double getFloat(int docID) {
final float f = values.get(docID);
// nocommit should we return NaN as default instead of 0.0?
return Float.isNaN(f) ? 0.0f : f;
@@ -290,9 +290,9 @@ public class Floats {
}
@Override
- public double floats(int docID) {
+ public double getFloat(int docID) {
final double d = values.get(docID);
- // nocommit should we return NaN as default instead of 0.0?
+ // TODO should we return NaN as default instead of 0.0?
return Double.isNaN(d) ? 0.0d : d;
}
@@ -302,7 +302,9 @@ public class Floats {
}
}
+ @Override
public void close() throws IOException {
+ super.close();
datIn.close();
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/MultiDocValues.java Fri Oct 29 20:33:36 2010
@@ -1,4 +1,5 @@
package org.apache.lucene.index.values;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,196 +17,214 @@ package org.apache.lucene.index.values;
* limitations under the License.
*/
import java.io.IOException;
-import java.util.List;
+import java.util.Arrays;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.MultiTermsEnum.TermsEnumIndex;
+import org.apache.lucene.index.values.DocValues.Source;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FloatsRef;
import org.apache.lucene.util.LongsRef;
-import org.apache.lucene.util.ReaderUtil.Slice;
+import org.apache.lucene.util.ReaderUtil;
public class MultiDocValues extends DocValues {
- public static class DocValuesIndex {
+ public static class DocValuesIndex { // nocommit is this necessary?
public final static DocValuesIndex[] EMPTY_ARRAY = new DocValuesIndex[0];
- final int subIndex;
+ final int start;
+ final int length;
final DocValues docValues;
- public DocValuesIndex(DocValues docValues, int subIndex) {
+ public DocValuesIndex(DocValues docValues, int start, int length) {
this.docValues = docValues;
- this.subIndex = subIndex;
+ this.start = start;
+ this.length = length;
}
}
private DocValuesIndex[] docValuesIdx;
- private Slice[] subSlices;
+ private int[] starts;
- public MultiDocValues(Slice[] subSlices) {
- this.subSlices = subSlices;
+ public MultiDocValues() {
+ starts = new int[0];
+ docValuesIdx = new DocValuesIndex[0];
}
- public MultiDocValues(DocValuesIndex[] docValuesIdx, Slice[] subSlices) {
- this(subSlices);
+ public MultiDocValues(DocValuesIndex[] docValuesIdx) {
reset(docValuesIdx);
}
@Override
public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new MultiValuesEnum(subSlices, docValuesIdx, docValuesIdx[0].docValues.type());
+ return new MultiValuesEnum(docValuesIdx, starts);
}
@Override
public Source load() throws IOException {
- return new MultiSource(subSlices, docValuesIdx);
+ return new MultiSource(docValuesIdx, starts);
}
public void close() throws IOException {
- //
+ super.close();
}
public DocValues reset(DocValuesIndex[] docValuesIdx) {
+ int[] start = new int[docValuesIdx.length];
+ for (int i = 0; i < docValuesIdx.length; i++) {
+ start[i] = docValuesIdx[i].start;
+ }
+ this.starts = start;
this.docValuesIdx = docValuesIdx;
return this;
}
+ public static class DummyDocValues extends DocValues {
+ final int maxDoc;
+ final Values type;
+ static final Source DUMMY = new DummySource();
+
+ public DummyDocValues(int maxDoc, Values type) {
+ this.type = type;
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
+ return new DummyEnum(attrSource, maxDoc, type);
+ }
+
+ @Override
+ public Source load() throws IOException {
+ return DUMMY;
+ }
+
+ @Override
+ public Source getCached(boolean load) throws IOException {
+ return DUMMY;
+ }
+
+ @Override
+ public Source releaseCached() {
+ return DUMMY;
+ }
+
+ @Override
+ public Values type() {
+ return type;
+ }
+
+ public void close() throws IOException {
+ super.close();
+ }
+
+ }
+
private static class MultiValuesEnum extends ValuesEnum {
- private int numDocs_ = 0;
- private int pos = -1;
- private int start = 0;
- private ValuesEnum current;
- private Slice[] subSlices;
private DocValuesIndex[] docValuesIdx;
private final int maxDoc;
-
- public MultiValuesEnum(Slice[] subSlices, DocValuesIndex[] docValuesIdx, Values type) {
- super(type);
- this.subSlices = subSlices;
+ private int currentStart;
+ private int currentMax;
+ private int currentDoc = -1;
+ private ValuesEnum currentEnum;
+ private final int[] starts;
+
+ public MultiValuesEnum(DocValuesIndex[] docValuesIdx, int[] starts)
+ throws IOException {
+ super(docValuesIdx[0].docValues.type());
this.docValuesIdx = docValuesIdx;
- Slice slice = subSlices[subSlices.length-1];
- maxDoc = slice.start + slice.length;
+ final DocValuesIndex last = docValuesIdx[docValuesIdx.length - 1];
+ maxDoc = last.start + last.length;
+ final DocValuesIndex idx = docValuesIdx[0];
+ currentEnum = idx.docValues.getEnum(this.attributes());
+ currentMax = idx.length;
+ currentStart = 0;
+ this.starts = starts;
}
@Override
public void close() throws IOException {
-
+ currentEnum.close();
}
@Override
public int advance(int target) throws IOException {
-// int n = target - start;
-// do {
-// if (target >= maxDoc)
-// return pos = NO_MORE_DOCS;
-// if (n >= numDocs_) {
-// int idx = readerIndex(target);
-// if (enumCache[idx] == null) {
-// try {
-// DocValues indexValues = subReaders[idx].docValues(id);
-// if (indexValues != null) // nocommit does that work with default
-// // values?
-// enumCache[idx] = indexValues.getEnum(this.attributes());
-// else
-// enumCache[idx] = new DummyEnum(this.attributes(),
-// subSlices[idx].length, attr.type());
-// } catch (IOException ex) {
-// // nocommit what to do here?
-// throw new RuntimeException(ex);
-// }
-// }
-// current = enumCache[idx];
-// start = subSlices[idx].start;
-// numDocs_ = subSlices[idx].length;
-// n = target - start;
-// }
-// target = start + numDocs_;
-// } while ((n = current.advance(n)) == NO_MORE_DOCS);
- return pos = start + current.docID();
+ assert target > currentDoc : "target " + target
+ + " must be > than the current doc " + currentDoc;
+ int relativeDoc = target - currentStart;
+ do {
+ if (target >= maxDoc) // we are beyond max doc
+ return currentDoc = NO_MORE_DOCS;
+ if (target >= currentMax) {
+ final int idx = ReaderUtil.subIndex(target, starts);
+ currentEnum.close();
+ currentEnum = docValuesIdx[idx].docValues.getEnum(this.attributes());
+ currentStart = docValuesIdx[idx].start;
+ currentMax = currentStart + docValuesIdx[idx].length;
+ relativeDoc = target - currentStart;
+ } else {
+ return currentDoc = currentStart + currentEnum.advance(relativeDoc);
+ }
+ } while ((relativeDoc = currentEnum.advance(relativeDoc)) == NO_MORE_DOCS);
+ return currentDoc = currentStart + relativeDoc;
}
@Override
public int docID() {
- return pos;
+ return currentDoc;
}
@Override
public int nextDoc() throws IOException {
- return advance(pos + 1);
+ return advance(currentDoc + 1);
}
}
- private class MultiSource extends Source {
- private int numDocs_ = 0;
+ private static class MultiSource extends Source {
+ private int numDocs = 0;
private int start = 0;
private Source current;
- private Slice[] subSlices;
- private DocValuesIndex[] docVAluesIdx;
+ private final int[] starts;
+ private final DocValuesIndex[] docValuesIdx;
- public MultiSource(Slice[] subSlices, DocValuesIndex[] docValuesIdx) {
- this.subSlices = subSlices;
- this.docVAluesIdx = docValuesIdx;
- }
-
- public long ints(int docID) {
-// int n = docID - start;
-// if (n >= numDocs_) {
-// int idx = readerIndex(docID);
-// try {
-// current = subReaders[idx].getIndexValuesCache().getInts(id);
-// if (current == null) // nocommit does that work with default values?
-// current = new DummySource();
-// } catch (IOException ex) {
-// // nocommit what to do here?
-// throw new RuntimeException(ex);
-// }
-// start = starts[idx];
-// numDocs_ = subReaders[idx].maxDoc();
-// n = docID - start;
-// }
-// return current.ints(n);
- return 0l;
- }
-
- public double floats(int docID) {
-// int n = docID - start;
-// if (n >= numDocs_) {
-// int idx = readerIndex(docID);
-// try {
-// current = subReaders[idx].getIndexValuesCache().getFloats(id);
-// if (current == null) // nocommit does that work with default values?
-// current = new DummySource();
-// } catch (IOException ex) {
-// // nocommit what to do here?
-// throw new RuntimeException(ex);
-// }
-// numDocs_ = subReaders[idx].maxDoc();
-//
-// start = starts[idx];
-// n = docID - start;
-// }
-// return current.floats(n);
- return 0d;
- }
-
- public BytesRef bytes(int docID) {
-// int n = docID - start;
-// if (n >= numDocs_) {
-// int idx = readerIndex(docID);
-// try {
-// current = subReaders[idx].getIndexValuesCache().getBytes(id);
-// if (current == null) // nocommit does that work with default values?
-// current = new DummySource();
-// } catch (IOException ex) {
-// // nocommit what to do here?
-// throw new RuntimeException(ex);
-// }
-// numDocs_ = subReaders[idx].maxDoc();
-// start = starts[idx];
-// n = docID - start;
-// }
-// return current.bytes(n);
- return null;
+ public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts) {
+ this.docValuesIdx = docValuesIdx;
+ this.starts = starts;
+
+ }
+
+ public long getInt(int docID) {
+ final int doc = ensureSource(docID);
+ return current.getInt(doc);
+ }
+
+ private final int ensureSource(int docID) {
+ int n = docID - start;
+ if (n >= numDocs) {
+ final int idx = ReaderUtil.subIndex(docID, starts);
+ assert idx >= 0 && idx < docValuesIdx.length : "idx was " + idx
+ + " for doc id: " + docID + " slices : " + Arrays.toString(starts);
+ assert docValuesIdx[idx] != null;
+ try {
+ current = docValuesIdx[idx].docValues.load();
+ } catch (IOException e) {
+ throw new RuntimeException("load failed", e); // TODO how should we
+ // handle this
+ }
+
+ start = docValuesIdx[idx].start;
+ numDocs = docValuesIdx[idx].length;
+ n = docID - start;
+ }
+ return n;
+ }
+
+ public double getFloat(int docID) {
+ final int doc = ensureSource(docID);
+ return current.getFloat(doc);
+ }
+
+ public BytesRef getBytes(int docID) {
+ final int doc = ensureSource(docID);
+ return current.getBytes(doc);
}
public long ramBytesUsed() {
@@ -218,17 +237,17 @@ public class MultiDocValues extends DocV
private final BytesRef ref = new BytesRef();
@Override
- public BytesRef bytes(int docID) {
+ public BytesRef getBytes(int docID) {
return ref;
}
@Override
- public double floats(int docID) {
+ public double getFloat(int docID) {
return 0.0d;
}
@Override
- public long ints(int docID) {
+ public long getInt(int docID) {
return 0;
}
@@ -296,5 +315,4 @@ public class MultiDocValues extends DocV
public Values type() {
return this.docValuesIdx[0].docValues.type();
}
-
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/PackedIntsImpl.java Fri Oct 29 20:33:36 2010
@@ -179,8 +179,8 @@ class PackedIntsImpl {
}
@Override
- public long ints(int docID) {
- // nocommit -- can we somehow avoid 2X method calls
+ public long getInt(int docID) {
+ // TODO -- can we somehow avoid 2X method calls
// on each get? must push minValue down, and make
// PackedInts implement Ints.Source
final long val = values.get(docID);
@@ -195,7 +195,9 @@ class PackedIntsImpl {
}
}
+ @Override
public void close() throws IOException {
+ super.close();
datIn.close();
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/Values.java Fri Oct 29 20:33:36 2010
@@ -34,7 +34,7 @@ public enum Values {
SIMPLE_FLOAT_4BYTE,
SIMPLE_FLOAT_8BYTE,
- // nocommit -- shouldn't lucene decide/detect straight vs
+ // TODO(simonw): -- shouldn't lucene decide/detect straight vs
// deref, as well fixed vs var?
BYTES_FIXED_STRAIGHT,
BYTES_FIXED_DEREF,
@@ -44,5 +44,5 @@ public enum Values {
BYTES_VAR_DEREF,
BYTES_VAR_SORTED
- // nocommit -- need STRING variants as well
+ // TODO(simonw): -- need STRING variants as well
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -135,7 +135,7 @@ class VarDerefBytesImpl {
idxOut.writeInt(address-1);
// write index
- // nocommit -- allow forcing fixed array (not -1)
+ // TODO(simonw): -- allow forcing fixed array (not -1)
// TODO(simonw): check the address calculation / make it more intuitive
final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, PackedInts.bitsRequired(address-1));
final int limit;
@@ -188,7 +188,7 @@ class VarDerefBytesImpl {
}
@Override
- public BytesRef bytes(int docID) {
+ public BytesRef getBytes(int docID) {
int address = (int) index.get(docID);
if (address == 0) {
assert defaultValue.length == 0: " default value manipulated";
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -120,7 +120,7 @@ class VarSortedBytesImpl {
idxOut.writeLong(offset);
// write index -- first doc -> 1+ord
- // nocommit -- allow not -1:
+ // TODO(simonw): allow not -1:
final PackedInts.Writer indexWriter = PackedInts.getWriter(idxOut,
docCount, PackedInts.bitsRequired(count));
final int limit = docCount > docToEntry.length ? docToEntry.length
@@ -135,7 +135,7 @@ class VarSortedBytesImpl {
indexWriter.finish();
// next ord (0-based) -> offset
- // nocommit -- allow not -1:
+ // TODO(simonw): -- allow not -1:
PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count,
PackedInts.bitsRequired(lastOffset));
for (int i = 0; i < count; i++) {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Fri Oct 29 20:33:36 2010
@@ -82,14 +82,15 @@ class VarStraightBytesImpl {
@Override
synchronized public void finish(int docCount) throws IOException {
- if (datOut == null)
+ if (datOut == null) {
return;
+ }
initIndexOut();
// write all lengths to index
// write index
fill(docCount);
idxOut.writeVInt(address);
- // nocommit -- allow not -1
+ // TODO(simonw): allow not -1
final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
PackedInts.bitsRequired(address));
for (int i = 0; i < docCount; i++) {
@@ -136,7 +137,7 @@ class VarStraightBytesImpl {
}
@Override
- public BytesRef bytes(int docID) {
+ public BytesRef getBytes(int docID) {
final int address = (int) addresses.get(docID);
bytesRef.offset = address;
if (docID == maxDoc - 1) {
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FieldComparator.java Fri Oct 29 20:33:36 2010
@@ -336,7 +336,7 @@ public abstract class FieldComparator {
@Override
public int compareBottom(int doc) {
- final double v2 = currentReaderValues.floats(doc);
+ final double v2 = currentReaderValues.getFloat(doc);
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
@@ -348,12 +348,12 @@ public abstract class FieldComparator {
@Override
public void copy(int slot, int doc) {
- values[slot] = currentReaderValues.floats(doc);
+ values[slot] = currentReaderValues.getFloat(doc);
}
@Override
public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = reader.getIndexValuesCache().getFloats(field);
+ currentReaderValues = reader.docValues(field).getCached(true);
return this;
}
@@ -538,7 +538,7 @@ public abstract class FieldComparator {
public int compareBottom(int doc) {
// TODO: there are sneaky non-branch ways to compute
// -1/+1/0 sign
- final long v2 = currentReaderValues.ints(doc);
+ final long v2 = currentReaderValues.getInt(doc);
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
@@ -550,12 +550,12 @@ public abstract class FieldComparator {
@Override
public void copy(int slot, int doc) {
- values[slot] = currentReaderValues.ints(doc);
+ values[slot] = currentReaderValues.getInt(doc);
}
@Override
public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException {
- currentReaderValues = reader.getIndexValuesCache().getInts(field);
+ currentReaderValues = reader.docValues(field).getCached(true);
return this;
}
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/SortField.java Fri Oct 29 20:33:36 2010
@@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.Sh
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.BytesRef;
-// nocommit -- for cleaner transition, maybe we should make
+// TODO(simonw) -- for cleaner transition, maybe we should make
// a new SortField that subclasses this one and always uses
// index values?
Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java Fri Oct 29 20:33:36 2010
@@ -173,4 +173,26 @@ public class ReaderUtil {
}
return hi;
}
+
+ public static int subIndex(int n, Slice[] slices) {
+ // searcher/reader for doc n:
+ int size = slices.length;
+ int lo = 0; // search starts array
+ int hi = size - 1; // for first element less than n, return its index
+ while (hi >= lo) {
+ int mid = (lo + hi) >>> 1;
+ int midValue = slices[mid].start;
+ if (n < midValue)
+ hi = mid - 1;
+ else if (n > midValue)
+ lo = mid + 1;
+ else { // found a match
+ while (mid + 1 < size && slices[mid + 1].start == midValue) {
+ mid++; // scan to last match
+ }
+ return mid;
+ }
+ }
+ return hi;
+ }
}
Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java?rev=1028899&r1=1028898&r2=1028899&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/values/TestIndexValues.java Fri Oct 29 20:33:36 2010
@@ -70,19 +70,19 @@ public class TestIndexValues extends Luc
public static void beforeClassLuceneTestCaseJ4() {
LuceneTestCase.beforeClassLuceneTestCaseJ4();
final CodecProvider cp = CodecProvider.getDefault();
- docValuesCodec = new DocValuesCodec(cp.lookup(CodecProvider.getDefaultCodec()));
+ docValuesCodec = new DocValuesCodec(cp.lookup(CodecProvider
+ .getDefaultCodec()));
cp.register(docValuesCodec);
CodecProvider.setDefaultCodec(docValuesCodec.name);
}
-
+
@AfterClass
public static void afterClassLuceneTestCaseJ4() {
final CodecProvider cp = CodecProvider.getDefault();
cp.unregister(docValuesCodec);
- LuceneTestCase.afterClassLuceneTestCaseJ4();
+ LuceneTestCase.afterClassLuceneTestCaseJ4();
}
-
-
+
public void testBytesStraight() throws IOException {
runTestBytes(Bytes.Mode.STRAIGHT, true);
runTestBytes(Bytes.Mode.STRAIGHT, false);
@@ -164,14 +164,14 @@ public class TestIndexValues extends Luc
if (mode == Bytes.Mode.SORTED) {
s = ss = r.loadSorted(comp);
} else {
- s = r.load();
+ s = getSource(r);
ss = null;
}
for (int i = 0; i < 100; i++) {
final int idx = 2 * i;
- assertNotNull("doc " + idx + "; value=" + values[idx], s.bytes(idx));
- assertEquals("doc " + idx, values[idx], s.bytes(idx).utf8ToString());
+ assertNotNull("doc " + idx + "; value=" + values[idx], s.getBytes(idx));
+ assertEquals("doc " + idx, values[idx], s.getBytes(idx).utf8ToString());
if (ss != null) {
assertEquals("doc " + idx, values[idx], ss.getByOrd(ss.ord(idx))
.utf8ToString());
@@ -247,9 +247,9 @@ public class TestIndexValues extends Luc
DocValues r = Ints.getValues(dir, "test", useFixedArrays);
for (int iter = 0; iter < 2; iter++) {
- Source s = r.load();
+ Source s = getSource(r);
for (int i = 0; i < NUM_VALUES; i++) {
- final long v = s.ints(i);
+ final long v = s.getInt(i);
assertEquals("index " + i + " b: " + b, values[i], v);
}
}
@@ -311,9 +311,9 @@ public class TestIndexValues extends Luc
DocValues r = Floats.getValues(dir, "test", NUM_VALUES + additionalValues);
for (int iter = 0; iter < 2; iter++) {
- Source s = r.load();
+ Source s = getSource(r);
for (int i = 0; i < NUM_VALUES; i++) {
- assertEquals(values[i], s.floats(i), 0.0f);
+ assertEquals(values[i], s.getFloat(i), 0.0f);
}
}
@@ -437,12 +437,12 @@ public class TestIndexValues extends Luc
case PACKED_INTS:
case PACKED_INTS_FIXED: {
DocValues intsReader = getDocValues(r, val.name());
- Source ints = intsReader.load();
+ Source ints = getSource(intsReader);
ValuesEnum intsEnum = intsReader.getEnum();
assertNotNull(intsEnum);
LongsRef enumRef = intsEnum.addAttribute(ValuesAttribute.class).ints();
for (int i = 0; i < base; i++) {
- assertEquals(0, ints.ints(i));
+ assertEquals(0, ints.getInt(i));
assertEquals(val.name() + " base: " + base + " index: " + i, i,
random.nextBoolean() ? intsEnum.advance(i) : intsEnum.nextDoc());
assertEquals(0, enumRef.get());
@@ -454,8 +454,8 @@ public class TestIndexValues extends Luc
}
assertEquals("advance failed at index: " + i + " of " + r.numDocs()
+ " docs", i, intsEnum.advance(i));
- assertEquals(expected, ints.ints(i));
assertEquals(expected, enumRef.get());
+ assertEquals(expected, ints.getInt(i));
}
}
@@ -463,14 +463,16 @@ public class TestIndexValues extends Luc
case SIMPLE_FLOAT_4BYTE:
case SIMPLE_FLOAT_8BYTE: {
DocValues floatReader = getDocValues(r, val.name());
- Source floats = floatReader.load();
+ assertNotNull(floatReader);
+ Source floats = getSource(floatReader);
ValuesEnum floatEnum = floatReader.getEnum();
assertNotNull(floatEnum);
FloatsRef enumRef = floatEnum.addAttribute(ValuesAttribute.class)
.floats();
for (int i = 0; i < base; i++) {
- assertEquals(0.0d, floats.floats(i), 0.0d);
+ assertEquals(" floats failed for doc: " + i + " base: " + base, 0.0d,
+ floats.getFloat(i), 0.0d);
assertEquals(i, random.nextBoolean() ? floatEnum.advance(i)
: floatEnum.nextDoc());
assertEquals("index " + i, 0.0, enumRef.get(), 0.0);
@@ -483,7 +485,8 @@ public class TestIndexValues extends Luc
assertEquals("advance failed at index: " + i + " of " + r.numDocs()
+ " docs base:" + base, i, floatEnum.advance(i));
assertEquals("index " + i, 2.0 * expected, enumRef.get(), 0.00001);
- assertEquals("index " + i, 2.0 * expected, floats.floats(i), 0.00001);
+ assertEquals("index " + i, 2.0 * expected, floats.getFloat(i),
+ 0.00001);
}
}
break;
@@ -505,15 +508,13 @@ public class TestIndexValues extends Luc
Values.BYTES_VAR_DEREF, Values.BYTES_VAR_SORTED,
Values.BYTES_VAR_STRAIGHT);
- private static EnumSet<Values> STRAIGHT_BYTES = EnumSet.of(
- Values.BYTES_FIXED_STRAIGHT, Values.BYTES_VAR_STRAIGHT);
-
private static EnumSet<Values> NUMERICS = EnumSet.of(Values.PACKED_INTS,
Values.PACKED_INTS_FIXED, Values.SIMPLE_FLOAT_4BYTE,
Values.SIMPLE_FLOAT_8BYTE);
private static Index[] IDX_VALUES = new Index[] { Index.ANALYZED,
- Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS };
+ Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS,
+ Index.NO };
private OpenBitSet indexValues(IndexWriter w, int numValues, Values value,
List<Values> valueVarList, boolean withDeletions, int multOfSeven)
@@ -521,9 +522,10 @@ public class TestIndexValues extends Luc
final boolean isNumeric = NUMERICS.contains(value);
OpenBitSet deleted = new OpenBitSet(numValues);
Document doc = new Document();
+ Index idx = IDX_VALUES[random.nextInt(IDX_VALUES.length)];
Fieldable field = random.nextBoolean() ? new ValuesField(value.name())
: newField(value.name(), _TestUtil.randomRealisticUnicodeString(random,
- 10), IDX_VALUES[random.nextInt(IDX_VALUES.length)]);
+ 10), idx == Index.NO ? Store.YES : Store.NO, idx);
doc.add(field);
ValuesAttribute valuesAttribute = ValuesField.values(field);
@@ -582,9 +584,10 @@ public class TestIndexValues extends Luc
}
w.commit();
- // nocommit test unoptimized with deletions
- if (true || withDeletions || random.nextBoolean())
- w.optimize();
+ // TODO test unoptimized with deletions
+ if (withDeletions || random.nextBoolean())
+ ;
+ w.optimize();
return deleted;
}
@@ -593,10 +596,9 @@ public class TestIndexValues extends Luc
Directory d = newDirectory();
IndexWriter w = new IndexWriter(d, cfg);
final List<Values> byteVariantList = new ArrayList<Values>(BYTES);
-
// run in random order to test if fill works correctly during merges
Collections.shuffle(byteVariantList, random);
- final int numValues = 350;
+ final int numValues = 333 + random.nextInt(150);
for (Values byteIndexValue : byteVariantList) {
List<Closeable> closeables = new ArrayList<Closeable>();
@@ -607,11 +609,10 @@ public class TestIndexValues extends Luc
assertEquals(0, r.numDeletedDocs());
final int numRemainingValues = (int) (numValues - deleted.cardinality());
final int base = r.numDocs() - numRemainingValues;
-
DocValues bytesReader = getDocValues(r, byteIndexValue.name());
assertNotNull("field " + byteIndexValue.name()
+ " returned null reader - maybe merged failed", bytesReader);
- Source bytes = bytesReader.load();
+ Source bytes = getSource(bytesReader);
ValuesEnum bytesEnum = bytesReader.getEnum();
assertNotNull(bytesEnum);
final ValuesAttribute attr = bytesEnum
@@ -619,7 +620,7 @@ public class TestIndexValues extends Luc
byte upto = 0;
// test the filled up slots for correctness
for (int i = 0; i < base; i++) {
- final BytesRef br = bytes.bytes(i);
+ final BytesRef br = bytes.getBytes(i);
String msg = " field: " + byteIndexValue.name() + " at index: " + i
+ " base: " + base + " numDocs:" + r.numDocs();
switch (byteIndexValue) {
@@ -645,7 +646,7 @@ public class TestIndexValues extends Luc
default:
assertNotNull("expected none null - " + msg, br);
if (br.length != 0) {
- bytes.bytes(i);
+ bytes.getBytes(i);
}
assertEquals("expected empty bytes - " + br.utf8ToString() + msg, 0,
br.length);
@@ -665,7 +666,7 @@ public class TestIndexValues extends Luc
upto += bytesSize;
}
- BytesRef br = bytes.bytes(i);
+ BytesRef br = bytes.getBytes(i);
if (bytesEnum.docID() != i)
assertEquals("seek failed for index " + i + " " + msg, i, bytesEnum
.advance(i));
@@ -692,10 +693,9 @@ public class TestIndexValues extends Luc
private DocValues getDocValues(IndexReader reader, String field)
throws IOException {
boolean optimized = reader.isOptimized();
- Fields fields = optimized ? reader.getSequentialSubReaders()[0].fields() : MultiFields
- .getFields(reader);
-// return fields.docValues(field);
- switch (random.nextInt(optimized ? 3 : 2)) {
+ Fields fields = optimized ? reader.getSequentialSubReaders()[0].fields()
+ : MultiFields.getFields(reader);
+ switch (random.nextInt(optimized ? 3 : 2)) { // case 2 only if optimized
case 0:
return fields.docValues(field);
case 1:
@@ -706,10 +706,14 @@ public class TestIndexValues extends Luc
return iterator.docValues();
}
throw new RuntimeException("no such field " + field);
- case 2:
+ case 2:// this only works if we are on an optimized index!
return reader.getSequentialSubReaders()[0].docValues(field);
}
-throw new RuntimeException();
-}
+ throw new RuntimeException();
+ }
+
+ private Source getSource(DocValues values) throws IOException {
+ return random.nextBoolean() ? values.load() : values.getCached(true);
+ }
}