You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/10/07 11:05:11 UTC
svn commit: r1179970 [2/3] - in /lucene/dev/trunk/lucene: ./
src/java/org/apache/lucene/document/ src/java/org/apache/lucene/index/
src/java/org/apache/lucene/index/codecs/
src/java/org/apache/lucene/index/codecs/sep/
src/java/org/apache/lucene/index/v...
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValuesArray.java Fri Oct 7 09:05:10 2011
@@ -1,12 +1,12 @@
package org.apache.lucene.index.values;
import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.Map;
-import org.apache.lucene.index.values.FixedStraightBytesImpl.FixedStraightBytesEnum;
import org.apache.lucene.index.values.IndexDocValues.Source;
-import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
@@ -32,82 +32,41 @@ import org.apache.lucene.util.RamUsageEs
*/
abstract class IndexDocValuesArray extends Source {
+ static final Map<ValueType, IndexDocValuesArray> TEMPLATES;
+
+ static {
+ EnumMap<ValueType, IndexDocValuesArray> templates = new EnumMap<ValueType, IndexDocValuesArray>(
+ ValueType.class);
+ templates.put(ValueType.FIXED_INTS_16, new ShortValues());
+ templates.put(ValueType.FIXED_INTS_32, new IntValues());
+ templates.put(ValueType.FIXED_INTS_64, new LongValues());
+ templates.put(ValueType.FIXED_INTS_8, new ByteValues());
+ templates.put(ValueType.FLOAT_32, new FloatValues());
+ templates.put(ValueType.FLOAT_64, new DoubleValues());
+ TEMPLATES = Collections.unmodifiableMap(templates);
+ }
+
protected final int bytesPerValue;
- private final ValueType type;
- private final boolean isFloat;
- protected int maxDocID = -1;
IndexDocValuesArray(int bytesPerValue, ValueType type) {
+ super(type);
this.bytesPerValue = bytesPerValue;
- this.type = type;
- switch (type) {
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- isFloat = false;
- break;
- case FLOAT_32:
- case FLOAT_64:
- isFloat = true;
- break;
- default:
- throw new IllegalStateException("illegal type: " + type);
-
- }
}
public abstract IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
throws IOException;
@Override
- public final int getValueCount() {
- return maxDocID + 1;
- }
-
- @Override
- public final ValueType type() {
- return type;
+ public final boolean hasArray() {
+ return true;
}
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- if (isFloat) {
- return new SourceEnum(attrSource, type(), this, maxDocID + 1) {
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs) {
- return pos = NO_MORE_DOCS;
- }
- floatsRef.floats[intsRef.offset] = IndexDocValuesArray.this
- .getFloat(target);
- return pos = target;
- }
- };
- } else {
- return new SourceEnum(attrSource, type(), this, maxDocID + 1) {
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs) {
- return pos = NO_MORE_DOCS;
- }
- intsRef.ints[intsRef.offset] = IndexDocValuesArray.this
- .getInt(target);
- return pos = target;
- }
-
- };
- }
+ void toBytes(long value, BytesRef bytesRef) {
+ bytesRef.copy(value);
}
- abstract ValuesEnum getDirectEnum(AttributeSource attrSource,
- IndexInput input, int maxDoc) throws IOException;
-
- @Override
- public final boolean hasArray() {
- return true;
+ void toBytes(double value, BytesRef bytesRef) {
+ bytesRef.copy(Double.doubleToRawLongBits(value));
}
final static class ByteValues extends IndexDocValuesArray {
@@ -122,7 +81,6 @@ abstract class IndexDocValuesArray exten
super(1, ValueType.FIXED_INTS_8);
values = new byte[numDocs];
input.readBytes(values, 0, values.length, false);
- maxDocID = numDocs - 1;
}
@Override
@@ -137,24 +95,15 @@ abstract class IndexDocValuesArray exten
}
@Override
- ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
- int maxDoc) throws IOException {
- return new FixedIntsEnum(attrSource, input, type(),
- bytesPerValue, maxDoc) {
-
- @Override
- protected final long toLong(BytesRef bytesRef) {
- return bytesRef.bytes[bytesRef.offset];
- }
- };
- }
-
- @Override
public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
throws IOException {
return new ByteValues(input, numDocs);
}
+ void toBytes(long value, BytesRef bytesRef) {
+ bytesRef.bytes[0] = (byte) (0xFFL & value);
+ }
+
};
final static class ShortValues extends IndexDocValuesArray {
@@ -171,7 +120,6 @@ abstract class IndexDocValuesArray exten
for (int i = 0; i < values.length; i++) {
values[i] = input.readShort();
}
- maxDocID = numDocs - 1;
}
@Override
@@ -186,24 +134,15 @@ abstract class IndexDocValuesArray exten
}
@Override
- ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
- int maxDoc) throws IOException {
- return new FixedIntsEnum(attrSource, input, type(),
- bytesPerValue, maxDoc) {
-
- @Override
- protected final long toLong(BytesRef bytesRef) {
- return bytesRef.asShort();
- }
- };
- }
-
- @Override
public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
throws IOException {
return new ShortValues(input, numDocs);
}
+ void toBytes(long value, BytesRef bytesRef) {
+ bytesRef.copy((short) (0xFFFFL & value));
+ }
+
};
final static class IntValues extends IndexDocValuesArray {
@@ -220,7 +159,6 @@ abstract class IndexDocValuesArray exten
for (int i = 0; i < values.length; i++) {
values[i] = input.readInt();
}
- maxDocID = numDocs - 1;
}
@Override
@@ -235,23 +173,15 @@ abstract class IndexDocValuesArray exten
}
@Override
- ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
- int maxDoc) throws IOException {
- return new FixedIntsEnum(attrSource, input, type(),
- bytesPerValue, maxDoc) {
- @Override
- protected final long toLong(BytesRef bytesRef) {
- return bytesRef.asInt();
- }
- };
- }
-
- @Override
public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
throws IOException {
return new IntValues(input, numDocs);
}
+ void toBytes(long value, BytesRef bytesRef) {
+ bytesRef.copy((int) (0xFFFFFFFF & value));
+ }
+
};
final static class LongValues extends IndexDocValuesArray {
@@ -268,7 +198,6 @@ abstract class IndexDocValuesArray exten
for (int i = 0; i < values.length; i++) {
values[i] = input.readLong();
}
- maxDocID = numDocs - 1;
}
@Override
@@ -283,18 +212,6 @@ abstract class IndexDocValuesArray exten
}
@Override
- ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
- int maxDoc) throws IOException {
- return new FixedIntsEnum(attrSource, input, type(),
- bytesPerValue, maxDoc) {
- @Override
- protected final long toLong(BytesRef bytesRef) {
- return bytesRef.asLong();
- }
- };
- }
-
- @Override
public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
throws IOException {
return new LongValues(input, numDocs);
@@ -313,13 +230,13 @@ abstract class IndexDocValuesArray exten
private FloatValues(IndexInput input, int numDocs) throws IOException {
super(RamUsageEstimator.NUM_BYTES_FLOAT, ValueType.FLOAT_32);
values = new float[numDocs];
- /* we always read BIG_ENDIAN here since the writer serialized plain bytes
- * we can simply read the ints / longs
- * back in using readInt / readLong */
+ /*
+ * we always read BIG_ENDIAN here since the writer serialized plain bytes
+ * we can simply read the ints / longs back in using readInt / readLong
+ */
for (int i = 0; i < values.length; i++) {
values[i] = Float.intBitsToFloat(input.readInt());
}
- maxDocID = numDocs - 1;
}
@Override
@@ -332,17 +249,11 @@ abstract class IndexDocValuesArray exten
assert docID >= 0 && docID < values.length;
return values[docID];
}
-
+
@Override
- ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
- int maxDoc) throws IOException {
- return new FloatsEnum(attrSource, input, type(),
- bytesPerValue, maxDoc) {
- @Override
- protected double toDouble(BytesRef bytesRef) {
- return Float.intBitsToFloat(bytesRef.asInt());
- }
- };
+ void toBytes(double value, BytesRef bytesRef) {
+ bytesRef.copy(Float.floatToRawIntBits((float)value));
+
}
@Override
@@ -351,7 +262,7 @@ abstract class IndexDocValuesArray exten
return new FloatValues(input, numDocs);
}
};
-
+
final static class DoubleValues extends IndexDocValuesArray {
private final double[] values;
@@ -363,13 +274,13 @@ abstract class IndexDocValuesArray exten
private DoubleValues(IndexInput input, int numDocs) throws IOException {
super(RamUsageEstimator.NUM_BYTES_DOUBLE, ValueType.FLOAT_64);
values = new double[numDocs];
- /* we always read BIG_ENDIAN here since the writer serialized plain bytes
- * we can simply read the ints / longs
- * back in using readInt / readLong */
+ /*
+ * we always read BIG_ENDIAN here since the writer serialized plain bytes
+ * we can simply read the ints / longs back in using readInt / readLong
+ */
for (int i = 0; i < values.length; i++) {
values[i] = Double.longBitsToDouble(input.readLong());
}
- maxDocID = numDocs - 1;
}
@Override
@@ -384,78 +295,11 @@ abstract class IndexDocValuesArray exten
}
@Override
- ValuesEnum getDirectEnum(AttributeSource attrSource, IndexInput input,
- int maxDoc) throws IOException {
- return new FloatsEnum(attrSource, input, type(),
- bytesPerValue, maxDoc) {
- @Override
- protected double toDouble(BytesRef bytesRef) {
- return Double.longBitsToDouble(bytesRef.asLong());
- }
- };
- }
-
- @Override
public IndexDocValuesArray newFromInput(IndexInput input, int numDocs)
throws IOException {
return new DoubleValues(input, numDocs);
}
- };
-
- private abstract static class FixedIntsEnum extends
- FixedStraightBytesEnum {
- private final ValueType type;
-
- private FixedIntsEnum(AttributeSource source, IndexInput dataIn,
- ValueType type, int bytesPerValue, int maxDoc) throws IOException {
- super(source, dataIn, bytesPerValue, maxDoc);
- this.type = type;
-
- }
- @Override
- public int advance(int target) throws IOException {
- final int advance = super.advance(target);
- if (advance != NO_MORE_DOCS) {
- intsRef.ints[0] = toLong(this.bytesRef);
- }
- return advance;
- }
-
- protected abstract long toLong(BytesRef bytesRef);
-
- @Override
- public ValueType type() {
- return type;
- }
-
- }
-
- private abstract static class FloatsEnum extends FixedStraightBytesEnum {
-
- private final ValueType type;
- FloatsEnum(AttributeSource source, IndexInput dataIn, ValueType type, int bytePerValue, int maxDoc)
- throws IOException {
- super(source, dataIn, bytePerValue, maxDoc);
- this.type = type;
- }
-
- @Override
- public int advance(int target) throws IOException {
- final int retVal = super.advance(target);
- if (retVal != NO_MORE_DOCS) {
- floatsRef.floats[floatsRef.offset] = toDouble(bytesRef);
- }
- return retVal;
- }
-
- protected abstract double toDouble(BytesRef bytesRef);
-
- @Override
- public ValueType type() {
- return type;
- }
-
- }
+ };
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Ints.java Fri Oct 7 09:05:10 2011
@@ -19,14 +19,9 @@ package org.apache.lucene.index.values;
import java.io.IOException;
-import org.apache.lucene.index.values.IndexDocValuesArray.ByteValues;
-import org.apache.lucene.index.values.IndexDocValuesArray.IntValues;
-import org.apache.lucene.index.values.IndexDocValuesArray.LongValues;
-import org.apache.lucene.index.values.IndexDocValuesArray.ShortValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IOUtils;
@@ -37,10 +32,13 @@ import org.apache.lucene.util.IOUtils;
* @lucene.experimental
*/
public final class Ints {
+ protected static final String CODEC_NAME = "Ints";
+ protected static final int VERSION_START = 0;
+ protected static final int VERSION_CURRENT = VERSION_START;
private Ints() {
}
-
+
public static Writer getWriter(Directory dir, String id, Counter bytesUsed,
ValueType type, IOContext context) throws IOException {
return type == ValueType.VAR_INTS ? new PackedIntValues.PackedIntsWriter(dir, id,
@@ -50,15 +48,42 @@ public final class Ints {
public static IndexDocValues getValues(Directory dir, String id, int numDocs,
ValueType type, IOContext context) throws IOException {
return type == ValueType.VAR_INTS ? new PackedIntValues.PackedIntsReader(dir, id,
- numDocs, context) : new IntsReader(dir, id, numDocs, context);
+ numDocs, context) : new IntsReader(dir, id, numDocs, context, type);
+ }
+
+ private static ValueType sizeToType(int size) {
+ switch (size) {
+ case 1:
+ return ValueType.FIXED_INTS_8;
+ case 2:
+ return ValueType.FIXED_INTS_16;
+ case 4:
+ return ValueType.FIXED_INTS_32;
+ case 8:
+ return ValueType.FIXED_INTS_64;
+ default:
+ throw new IllegalStateException("illegal size " + size);
+ }
+ }
+
+ private static int typeToSize(ValueType type) {
+ switch (type) {
+ case FIXED_INTS_16:
+ return 2;
+ case FIXED_INTS_32:
+ return 4;
+ case FIXED_INTS_64:
+ return 8;
+ case FIXED_INTS_8:
+ return 1;
+ default:
+ throw new IllegalStateException("illegal type " + type);
+ }
}
- static class IntsWriter extends FixedStraightBytesImpl.Writer {
- protected static final String CODEC_NAME = "Ints";
- protected static final int VERSION_START = 0;
- protected static final int VERSION_CURRENT = VERSION_START;
- private final ValueType valueType;
+ static class IntsWriter extends FixedStraightBytesImpl.Writer {
+ private final IndexDocValuesArray template;
public IntsWriter(Directory dir, String id, Counter bytesUsed,
IOContext context, ValueType valueType) throws IOException {
@@ -68,46 +93,15 @@ public final class Ints {
protected IntsWriter(Directory dir, String id, String codecName,
int version, Counter bytesUsed, IOContext context, ValueType valueType) throws IOException {
super(dir, id, codecName, version, bytesUsed, context);
- this.valueType = valueType;
- final int expectedSize = getSize(valueType);
+ final int expectedSize = typeToSize(valueType);
this.bytesRef = new BytesRef(expectedSize);
bytesRef.length = expectedSize;
+ template = IndexDocValuesArray.TEMPLATES.get(valueType);
}
- private static int getSize(ValueType type) {
- switch (type) {
- case FIXED_INTS_16:
- return 2;
- case FIXED_INTS_32:
- return 4;
- case FIXED_INTS_64:
- return 8;
- case FIXED_INTS_8:
- return 1;
- default:
- throw new IllegalStateException("illegal type " + type);
- }
- }
-
@Override
public void add(int docID, long v) throws IOException {
- switch (valueType) {
- case FIXED_INTS_64:
- bytesRef.copy(v);
- break;
- case FIXED_INTS_32:
- bytesRef.copy((int) (0xFFFFFFFF & v));
- break;
- case FIXED_INTS_16:
- bytesRef.copy((short) (0xFFFFL & v));
- break;
- case FIXED_INTS_8:
- bytesRef.bytes[0] = (byte) (0xFFL & v);
- break;
- default:
- throw new IllegalStateException("illegal type " + valueType);
- }
-
+ template.toBytes(v, bytesRef);
add(docID, bytesRef);
}
@@ -116,72 +110,27 @@ public final class Ints {
add(docID, docValues.getInt());
}
}
-
- final static class IntsReader extends FixedStraightBytesImpl.Reader {
- private final ValueType type;
+
+ final static class IntsReader extends FixedStraightBytesImpl.FixedStraightReader {
private final IndexDocValuesArray arrayTemplate;
- IntsReader(Directory dir, String id, int maxDoc, IOContext context)
+ IntsReader(Directory dir, String id, int maxDoc, IOContext context, ValueType type)
throws IOException {
- super(dir, id, IntsWriter.CODEC_NAME, IntsWriter.VERSION_CURRENT, maxDoc,
- context);
- switch (size) {
- case 8:
- type = ValueType.FIXED_INTS_64;
- arrayTemplate = new LongValues();
- break;
- case 4:
- type = ValueType.FIXED_INTS_32;
- arrayTemplate = new IntValues();
- break;
- case 2:
- type = ValueType.FIXED_INTS_16;
- arrayTemplate = new ShortValues();
- break;
- case 1:
- type = ValueType.FIXED_INTS_8;
- arrayTemplate = new ByteValues();
- break;
- default:
- throw new IllegalStateException("illegal size: " + size);
- }
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, maxDoc,
+ context, type);
+ arrayTemplate = IndexDocValuesArray.TEMPLATES.get(type);
+ assert arrayTemplate != null;
+ assert type == sizeToType(size);
}
@Override
public Source load() throws IOException {
- boolean success = false;
- IndexInput input = null;
+ final IndexInput indexInput = cloneData();
try {
- input = cloneData();
- final Source source = arrayTemplate.newFromInput(input, maxDoc);
- success = true;
- return source;
+ return arrayTemplate.newFromInput(indexInput, maxDoc);
} finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(input, datIn);
- }
+ IOUtils.close(indexInput);
}
}
-
- @Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- final IndexInput input = cloneData();
- boolean success = false;
- try {
- final ValuesEnum valuesEnum = arrayTemplate.getDirectEnum(source,
- input, maxDoc);
- success = true;
- return valuesEnum;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(input);
- }
- }
- }
-
- @Override
- public ValueType type() {
- return type;
- }
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java Fri Oct 7 09:05:10 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
@@ -28,6 +27,7 @@ import org.apache.lucene.util.ReaderUtil
* {@link IndexDocValues}
*
* @lucene.experimental
+ * @lucene.internal
*/
public class MultiIndexDocValues extends IndexDocValues {
@@ -57,13 +57,8 @@ public class MultiIndexDocValues extends
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new MultiValuesEnum(docValuesIdx, starts);
- }
-
- @Override
public Source load() throws IOException {
- return new MultiSource(docValuesIdx, starts);
+ return new MultiSource(docValuesIdx, starts, false);
}
public IndexDocValues reset(DocValuesIndex[] docValuesIdx) {
@@ -86,11 +81,6 @@ public class MultiIndexDocValues extends
}
@Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return emptySoruce.getEnum(attrSource);
- }
-
- @Override
public Source load() throws IOException {
return emptySoruce;
}
@@ -99,69 +89,11 @@ public class MultiIndexDocValues extends
public ValueType type() {
return emptySoruce.type();
}
- }
-
- private static class MultiValuesEnum extends ValuesEnum {
- private DocValuesIndex[] docValuesIdx;
- private final int maxDoc;
- private int currentStart;
- private int currentMax;
- private int currentDoc = -1;
- private ValuesEnum currentEnum;
- private final int[] starts;
-
- public MultiValuesEnum(DocValuesIndex[] docValuesIdx, int[] starts)
- throws IOException {
- super(docValuesIdx[0].docValues.type());
- this.docValuesIdx = docValuesIdx;
- final DocValuesIndex last = docValuesIdx[docValuesIdx.length - 1];
- maxDoc = last.start + last.length;
- final DocValuesIndex idx = docValuesIdx[0];
- currentEnum = idx.docValues.getEnum(this.attributes());
- currentEnum.copyFrom(this);
- intsRef = currentEnum.intsRef;
- currentMax = idx.length;
- currentStart = 0;
- this.starts = starts;
- }
-
- @Override
- public void close() throws IOException {
- currentEnum.close();
- }
-
- @Override
- public int advance(int target) throws IOException {
- assert target > currentDoc : "target " + target
- + " must be > than the current doc " + currentDoc;
- int relativeDoc = target - currentStart;
- do {
- if (target >= maxDoc) {// we are beyond max doc
- return currentDoc = NO_MORE_DOCS;
- }
- if (target >= currentMax) {
- final int idx = ReaderUtil.subIndex(target, starts);
- currentEnum.close();
- currentEnum = docValuesIdx[idx].docValues.getEnum();
- currentEnum.copyFrom(this);
- currentStart = docValuesIdx[idx].start;
- currentMax = currentStart + docValuesIdx[idx].length;
- relativeDoc = target - currentStart;
- }
- target = currentMax; // make sure that we advance to the next enum if the current is exhausted
-
- } while ((relativeDoc = currentEnum.advance(relativeDoc)) == NO_MORE_DOCS);
- return currentDoc = currentStart + relativeDoc;
- }
- @Override
- public int docID() {
- return currentDoc;
- }
@Override
- public int nextDoc() throws IOException {
- return advance(currentDoc + 1);
+ public Source getDirectSource() throws IOException {
+ return emptySoruce;
}
}
@@ -171,12 +103,14 @@ public class MultiIndexDocValues extends
private Source current;
private final int[] starts;
private final DocValuesIndex[] docValuesIdx;
+ private boolean direct;
- public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts) {
+ public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts, boolean direct) {
+ super(docValuesIdx[0].docValues.type());
this.docValuesIdx = docValuesIdx;
this.starts = starts;
assert docValuesIdx.length != 0;
-
+ this.direct = direct;
}
public long getInt(int docID) {
@@ -193,7 +127,11 @@ public class MultiIndexDocValues extends
+ " for doc id: " + docID + " slices : " + Arrays.toString(starts);
assert docValuesIdx[idx] != null;
try {
- current = docValuesIdx[idx].docValues.getSource();
+ if (direct) {
+ current = docValuesIdx[idx].docValues.getDirectSource();
+ } else {
+ current = docValuesIdx[idx].docValues.getSource();
+ }
} catch (IOException e) {
throw new RuntimeException("load failed", e); // TODO how should we
// handle this
@@ -214,24 +152,12 @@ public class MultiIndexDocValues extends
final int doc = ensureSource(docID);
return current.getBytes(doc, bytesRef);
}
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- throw new UnsupportedOperationException(); // TODO
- }
-
- @Override
- public ValueType type() {
- return docValuesIdx[0].docValues.type();
- }
-
}
private static class EmptySource extends Source {
- private final ValueType type;
public EmptySource(ValueType type) {
- this.type = type;
+ super(type);
}
@Override
@@ -250,20 +176,15 @@ public class MultiIndexDocValues extends
public long getInt(int docID) {
return 0;
}
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return ValuesEnum.emptyEnum(type);
- }
-
- @Override
- public ValueType type() {
- return type;
- }
}
@Override
public ValueType type() {
return this.docValuesIdx[0].docValues.type();
}
+
+ @Override
+ public Source getDirectSource() throws IOException {
+ return new MultiSource(docValuesIdx, starts, true);
+ }
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java Fri Oct 7 09:05:10 2011
@@ -21,18 +21,15 @@ import java.io.IOException;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.values.FixedStraightBytesImpl.FixedBytesWriterBase;
import org.apache.lucene.index.values.IndexDocValues.Source;
-import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
import org.apache.lucene.index.values.IndexDocValuesArray.LongValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.packed.PackedInts;
/**
@@ -51,7 +48,6 @@ class PackedIntValues {
static class PackedIntsWriter extends FixedBytesWriterBase {
- private LongsRef intsRef;
private long minValue;
private long maxValue;
private boolean started;
@@ -114,10 +110,10 @@ class PackedIntValues {
}
@Override
- protected void mergeDoc(int docID) throws IOException {
+ protected void mergeDoc(int docID, int sourceDoc) throws IOException {
assert docID > lastDocId : "docID: " + docID
+ " must be greater than the last added doc id: " + lastDocId;
- add(docID, intsRef.get());
+ add(docID, currentMergeSource.getInt(sourceDoc));
}
private void writePackedInts(IndexOutput datOut, int docCount) throws IOException {
@@ -139,12 +135,6 @@ class PackedIntValues {
w.add(defaultValue);
}
w.finish();
- w.finish();
- }
-
- @Override
- protected void setNextEnum(ValuesEnum valuesEnum) {
- intsRef = valuesEnum.getInt();
}
@Override
@@ -215,30 +205,17 @@ class PackedIntValues {
datIn.close();
}
- @Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- final IndexInput input = (IndexInput) datIn.clone();
- boolean success = false;
- try {
- final ValuesEnum inst;
- if (values == null) {
- inst = new PackedIntsEnumImpl(source, input);
- } else {
- inst = values.getDirectEnum(source, input, numDocs);
- }
- success = true;
- return inst;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(input);
- }
- }
- }
@Override
public ValueType type() {
return ValueType.VAR_INTS;
}
+
+
+ @Override
+ public Source getDirectSource() throws IOException {
+ return values != null ? new FixedStraightBytesImpl.DirectFixedStraightSource((IndexInput) datIn.clone(), 8, ValueType.FIXED_INTS_64) : new DirectPackedIntsSource((IndexInput) datIn.clone());
+ }
}
@@ -248,7 +225,7 @@ class PackedIntValues {
private final PackedInts.Reader values;
public PackedIntsSource(IndexInput dataIn) throws IOException {
-
+ super(ValueType.VAR_INTS);
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
values = PackedInts.getReader(dataIn);
@@ -263,72 +240,41 @@ class PackedIntValues {
final long value = values.get(docID);
return value == defaultValue ? 0 : minValue + value;
}
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return new SourceEnum(attrSource, type(), this, values.size()) {
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs)
- return pos = NO_MORE_DOCS;
- intsRef.ints[intsRef.offset] = source.getInt(target);
- return pos = target;
- }
- };
- }
-
- @Override
- public ValueType type() {
- return ValueType.VAR_INTS;
- }
}
- private static final class PackedIntsEnumImpl extends ValuesEnum {
- private final PackedInts.ReaderIterator ints;
+ private static final class DirectPackedIntsSource extends Source {
+ private final PackedInts.RandomAccessReaderIterator ints;
private long minValue;
- private final IndexInput dataIn;
private final long defaultValue;
- private final int maxDoc;
- private int pos = -1;
- private PackedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
+ private DirectPackedIntsSource(IndexInput dataIn)
throws IOException {
- super(source, ValueType.VAR_INTS);
- intsRef.offset = 0;
- this.dataIn = dataIn;
+ super(ValueType.VAR_INTS);
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
- this.ints = PackedInts.getReaderIterator(dataIn);
- maxDoc = ints.size();
+ this.ints = PackedInts.getRandomAccessReaderIterator(dataIn);
}
@Override
- public void close() throws IOException {
- ints.close();
- dataIn.close();
+ public double getFloat(int docID) {
+ return getInt(docID);
}
@Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- final long val = ints.advance(target);
- intsRef.ints[intsRef.offset] = val == defaultValue ? 0 : minValue + val;
- return pos = target;
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ ref.grow(8);
+ ref.copy(getInt(docID));
+ return ref;
}
@Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= maxDoc) {
- return pos = NO_MORE_DOCS;
+ public long getInt(int docID) {
+ try {
+ final long val = ints.get(docID);
+ return val == defaultValue ? 0 : minValue + val;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
- return advance(pos + 1);
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/SourceCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/SourceCache.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/SourceCache.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/SourceCache.java Fri Oct 7 09:05:10 2011
@@ -18,36 +18,29 @@ package org.apache.lucene.index.values;
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.values.IndexDocValues.SortedSource;
import org.apache.lucene.index.values.IndexDocValues.Source;
-import org.apache.lucene.util.BytesRef;
/**
- * Abstract base class for {@link IndexDocValues} {@link Source} /
- * {@link SortedSource} cache.
+ * Abstract base class for {@link IndexDocValues} {@link Source} cache.
* <p>
- * {@link Source} and {@link SortedSource} instances loaded via
- * {@link IndexDocValues#load()} and {@link IndexDocValues#loadSorted(Comparator)} are
- * entirely memory resident and need to be maintained by the caller. Each call
- * to {@link IndexDocValues#load()} or {@link IndexDocValues#loadSorted(Comparator)} will
- * cause an entire reload of the underlying data. Source and
- * {@link SortedSource} instances obtained from {@link IndexDocValues#getSource()}
- * and {@link IndexDocValues#getSource()} respectively are maintained by a
- * {@link SourceCache} that is closed ({@link #close(IndexDocValues)}) once the
- * {@link IndexReader} that created the {@link IndexDocValues} instance is closed.
+ * {@link Source} instances loaded via {@link IndexDocValues#load()} are entirely memory resident
+ * and need to be maintained by the caller. Each call to
+ * {@link IndexDocValues#load()} will cause an entire reload of
+ * the underlying data. Source instances obtained from
+ * {@link IndexDocValues#getSource()} and {@link IndexDocValues#getSource()}
+ * respectively are maintained by a {@link SourceCache} that is closed (
+ * {@link #close(IndexDocValues)}) once the {@link IndexReader} that created the
+ * {@link IndexDocValues} instance is closed.
* <p>
- * Unless {@link Source} and {@link SortedSource} instances are managed by
- * another entity it is recommended to use the cached variants to obtain a
- * source instance.
+ * Unless {@link Source} instances are managed by another entity it is
+ * recommended to use the cached variants to obtain a source instance.
* <p>
* Implementation of this API must be thread-safe.
*
* @see IndexDocValues#setCache(SourceCache)
* @see IndexDocValues#getSource()
- * @see IndexDocValues#getSortedSorted(Comparator)
*
* @lucene.experimental
*/
@@ -63,17 +56,7 @@ public abstract class SourceCache {
public abstract Source load(IndexDocValues values) throws IOException;
/**
- * Atomically loads a {@link SortedSource} into the cache from the given
- * {@link IndexDocValues} and returns it iff no other {@link SortedSource} has
- * already been cached. Otherwise the cached source is returned.
- * <p>
- * This method will not return <code>null</code>
- */
- public abstract SortedSource loadSorted(IndexDocValues values,
- Comparator<BytesRef> comp) throws IOException;
-
- /**
- * Atomically invalidates the cached {@link Source} and {@link SortedSource}
+ * Atomically invalidates the cached {@link Source}
* instances if any and empties the cache.
*/
public abstract void invalidate(IndexDocValues values);
@@ -87,14 +70,13 @@ public abstract class SourceCache {
/**
* Simple per {@link IndexDocValues} instance cache implementation that holds a
- * {@link Source} and {@link SortedSource} reference as a member variable.
+ * {@link Source} a member variable.
* <p>
* If a {@link DirectSourceCache} instance is closed or invalidated the cached
* reference are simply set to <code>null</code>
*/
public static final class DirectSourceCache extends SourceCache {
private Source ref;
- private SortedSource sortedRef;
public synchronized Source load(IndexDocValues values) throws IOException {
if (ref == null) {
@@ -103,17 +85,8 @@ public abstract class SourceCache {
return ref;
}
- public synchronized SortedSource loadSorted(IndexDocValues values,
- Comparator<BytesRef> comp) throws IOException {
- if (sortedRef == null) {
- sortedRef = values.loadSorted(comp);
- }
- return sortedRef;
- }
-
public synchronized void invalidate(IndexDocValues values) {
ref = null;
- sortedRef = null;
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/ValueType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/ValueType.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/ValueType.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/ValueType.java Fri Oct 7 09:05:10 2011
@@ -43,9 +43,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
VAR_INTS,
@@ -56,9 +55,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_8,
@@ -69,9 +67,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_16,
@@ -82,9 +79,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_32,
@@ -95,9 +91,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_64,
@@ -110,9 +105,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0.0f</tt> as the default value without any
* distinction between provided <tt>0.0f</tt> values during indexing. All
- * documents without an explicit value will use <tt>0.0f</tt> instead. In
- * turn, {@link ValuesEnum} instances will not skip documents without an
- * explicit value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0.0f</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FLOAT_32,
@@ -126,9 +120,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0.0d</tt> as the default value without any
* distinction between provided <tt>0.0d</tt> values during indexing. All
- * documents without an explicit value will use <tt>0.0d</tt> instead. In
- * turn, {@link ValuesEnum} instances will not skip documents without an
- * explicit value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0.0d</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FLOAT_64,
@@ -143,9 +136,7 @@ public enum ValueType {
* NOTE: this type uses <tt>0 byte</tt> filled byte[] based on the length of the first seen
* value as the default value without any distinction between explicitly
* provided values during indexing. All documents without an explicit value
- * will use the default instead. In turn, {@link ValuesEnum} instances will
- * not skip documents without an explicit value assigned. Custom default
- * values must be assigned explicitly.
+ * will use the default instead.Custom default values must be assigned explicitly.
* </p>
*/
BYTES_FIXED_STRAIGHT,
@@ -159,34 +150,12 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without and
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
- * explicitly.
+ * reference. Custom default values must be assigned explicitly.
* </p>
*/
BYTES_FIXED_DEREF,
/**
- * A fixed length pre-sorted byte[] variant. Fields with this type only
- * store distinct byte values and store an additional offset pointer per
- * document to dereference the shared byte[]. The stored
- * byte[] is presorted, by default by unsigned byte order,
- * and allows access via document id, ordinal and by-value.
- * Use this type if your documents may share the same byte[].
- * <p>
- * NOTE: Fields of this type will not store values for documents without and
- * explicitly provided value. If a documents value is accessed while no
- * explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
- * explicitly.
- * </p>
- *
- * @see SortedSource
- */
- BYTES_FIXED_SORTED,
-
- /**
* Variable length straight stored byte[] variant. All bytes are
* stored sequentially for compactness. Usage of this type via the
* disk-resident API might yield performance degradation since no additional
@@ -195,9 +164,7 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without an
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * byte[] reference. In contrast to dereferenced variants, {@link ValuesEnum}
- * instances will <b>not</b> skip over documents without an explicit value
- * assigned. Custom default values must be assigned explicitly.
+ * byte[] reference. Custom default values must be assigned explicitly.
* </p>
*/
BYTES_VAR_STRAIGHT,
@@ -210,13 +177,12 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without and
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
- * explicitly.
+ * reference. Custom default values must be assigned explicitly.
* </p>
*/
BYTES_VAR_DEREF,
+
/**
* A variable length pre-sorted byte[] variant. Just like
* {@link #BYTES_FIXED_SORTED}, but allowing each
@@ -225,12 +191,30 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without and
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
+ * reference.Custom default values must be assigned explicitly.
+ * </p>
+ *
+ * @see SortedSource
+ */
+ BYTES_VAR_SORTED,
+
+ /**
+ * A fixed length pre-sorted byte[] variant. Fields with this type only
+ * store distinct byte values and store an additional offset pointer per
+ * document to dereference the shared byte[]. The stored
+ * byte[] is presorted, by default by unsigned byte order,
+ * and allows access via document id, ordinal and by-value.
+ * Use this type if your documents may share the same byte[].
+ * <p>
+ * NOTE: Fields of this type will not store values for documents without and
+ * explicitly provided value. If a documents value is accessed while no
+ * explicit value is stored the returned {@link BytesRef} will be a 0-length
+ * reference. Custom default values must be assigned
* explicitly.
* </p>
*
* @see SortedSource
*/
- BYTES_VAR_SORTED
+ BYTES_FIXED_SORTED
+
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Fri Oct 7 09:05:10 2011
@@ -20,16 +20,17 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesEnumBase;
+import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
+import org.apache.lucene.index.values.DirectSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.PackedInts;
// Stores variable-length byte[] by deref, ie when two docs
// have the same value, they store only 1 byte[] and both
@@ -57,6 +58,7 @@ class VarDerefBytesImpl {
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ size = 0;
}
@Override
@@ -68,88 +70,82 @@ class VarDerefBytesImpl {
// some last docs that we didn't see
@Override
public void finishInternal(int docCount) throws IOException {
+ fillDefault(docCount);
final int size = hash.size();
- final long[] addresses = new long[size+1];
+ final long[] addresses = new long[size];
final IndexOutput datOut = getOrCreateDataOut();
- int addr = 1;
+ int addr = 0;
final BytesRef bytesRef = new BytesRef();
for (int i = 0; i < size; i++) {
hash.get(i, bytesRef);
- addresses[i+1] = addr;
+ addresses[i] = addr;
addr += writePrefixLength(datOut, bytesRef) + bytesRef.length;
datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
final IndexOutput idxOut = getOrCreateIndexOut();
// write the max address to read directly on source load
- idxOut.writeLong(addr - 1);
- writeIndex(idxOut, docCount, addresses[size], addresses, docToEntry);
+ idxOut.writeLong(addr);
+ writeIndex(idxOut, docCount, addresses[addresses.length-1], addresses, docToEntry);
}
}
- public static class Reader extends BytesReaderBase {
+ public static class VarDerefReader extends BytesReaderBase {
private final long totalBytes;
- Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true, context);
+ VarDerefReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context, ValueType.BYTES_VAR_DEREF);
totalBytes = idxIn.readLong();
}
@Override
public Source load() throws IOException {
- return new Source(cloneData(), cloneIndex(), totalBytes);
+ return new VarDerefSource(cloneData(), cloneIndex(), totalBytes);
}
+
+ @Override
+ public Source getDirectSource()
+ throws IOException {
+ return new DirectVarDerefSource(cloneData(), cloneIndex(), type());
+ }
+ }
+
+ final static class VarDerefSource extends BytesSourceBase {
+ private final PackedInts.Reader addresses;
- private final static class Source extends DerefBytesSourceBase {
-
- public Source(IndexInput datIn, IndexInput idxIn, long totalBytes)
- throws IOException {
- super(datIn, idxIn, totalBytes, ValueType.BYTES_VAR_DEREF);
- }
-
- @Override
- public BytesRef getBytes(int docID, BytesRef bytesRef) {
- long address = addresses.get(docID);
- bytesRef.length = 0;
- return address == 0 ? bytesRef : data.fillSliceWithPrefix(bytesRef,
- --address);
- }
+ public VarDerefSource(IndexInput datIn, IndexInput idxIn, long totalBytes)
+ throws IOException {
+ super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), totalBytes,
+ ValueType.BYTES_VAR_DEREF);
+ addresses = PackedInts.getReader(idxIn);
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new VarDerefBytesEnum(source, cloneData(), cloneIndex());
+ public BytesRef getBytes(int docID, BytesRef bytesRef) {
+ return data.fillSliceWithPrefix(bytesRef,
+ addresses.get(docID));
}
+ }
- final static class VarDerefBytesEnum extends DerefBytesEnumBase {
-
- public VarDerefBytesEnum(AttributeSource source, IndexInput datIn,
- IndexInput idxIn) throws IOException {
- super(source, datIn, idxIn, -1, ValueType.BYTES_VAR_DEREF);
- }
+
+ final static class DirectVarDerefSource extends DirectSource {
+ private final PackedInts.RandomAccessReaderIterator index;
- @Override
- protected void fill(long address, BytesRef ref) throws IOException {
- datIn.seek(fp + --address);
- final byte sizeByte = datIn.readByte();
- final int size;
- if ((sizeByte & 128) == 0) {
- // length is 1 byte
- size = sizeByte;
- } else {
- size = ((sizeByte & 0x7f) << 8) | ((datIn.readByte() & 0xff));
- }
- if (ref.bytes.length < size) {
- ref.grow(size);
- }
- ref.length = size;
- ref.offset = 0;
- datIn.readBytes(ref.bytes, 0, size);
- }
+ DirectVarDerefSource(IndexInput data, IndexInput index, ValueType type)
+ throws IOException {
+ super(data, type);
+ this.index = PackedInts.getRandomAccessReaderIterator(index);
}
-
+
@Override
- public ValueType type() {
- return ValueType.BYTES_VAR_DEREF;
+ protected int position(int docID) throws IOException {
+ data.seek(baseOffset + index.get(docID));
+ final byte sizeByte = data.readByte();
+ if ((sizeByte & 128) == 0) {
+ // length is 1 byte
+ return sizeByte;
+ } else {
+ return ((sizeByte & 0x7f) << 8) | ((data.readByte() & 0xff));
+ }
}
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Fri Oct 7 09:05:10 2011
@@ -23,11 +23,11 @@ import java.util.Comparator;
import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
+import org.apache.lucene.index.values.IndexDocValues.SortedSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.packed.PackedInts;
@@ -39,7 +39,7 @@ import org.apache.lucene.util.packed.Pac
/**
* @lucene.experimental
*/
-class VarSortedBytesImpl {
+final class VarSortedBytesImpl {
static final String CODEC_NAME = "VarDerefBytes";
static final int VERSION_START = 0;
@@ -52,8 +52,9 @@ class VarSortedBytesImpl {
Counter bytesUsed, IOContext context) throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
this.comp = comp;
+ size = 0;
}
-
+
@Override
protected void checkSize(BytesRef bytes) {
// allow var bytes sizes
@@ -63,11 +64,11 @@ class VarSortedBytesImpl {
// some last docs that we didn't see
@Override
public void finishInternal(int docCount) throws IOException {
+ fillDefault(docCount);
final int count = hash.size();
final IndexOutput datOut = getOrCreateDataOut();
long offset = 0;
- long lastOffset = 0;
- final int[] index = new int[count+1];
+ final int[] index = new int[count];
final long[] offsets = new long[count];
final int[] sortedEntries = hash.sort(comp);
// first dump bytes data, recording index & offset as
@@ -75,173 +76,125 @@ class VarSortedBytesImpl {
for (int i = 0; i < count; i++) {
final int e = sortedEntries[i];
offsets[i] = offset;
- index[e+1] = 1 + i;
+ index[e] = i;
final BytesRef bytes = hash.get(e, new BytesRef());
// TODO: we could prefix code...
datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
- lastOffset = offset;
offset += bytes.length;
}
-
final IndexOutput idxOut = getOrCreateIndexOut();
// total bytes of data
idxOut.writeLong(offset);
- // write index -- first doc -> 1+ord
+ // write index
writeIndex(idxOut, docCount, count, index, docToEntry);
// next ord (0-based) -> offset
- PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count,
- PackedInts.bitsRequired(lastOffset));
+ PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count+1,
+ PackedInts.bitsRequired(offset));
for (int i = 0; i < count; i++) {
offsetWriter.add(offsets[i]);
}
+ offsetWriter.add(offset);
offsetWriter.finish();
}
}
public static class Reader extends BytesReaderBase {
- private final Comparator<BytesRef> defaultComp;
+ private final Comparator<BytesRef> comparator;
Reader(Directory dir, String id, int maxDoc,
- Comparator<BytesRef> comparator, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true, context);
- this.defaultComp = comparator;
+ IOContext context, ValueType type, Comparator<BytesRef> comparator)
+ throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context, type);
+ this.comparator = comparator;
}
@Override
public org.apache.lucene.index.values.IndexDocValues.Source load()
throws IOException {
- return loadSorted(defaultComp);
+ return new VarSortedSource(cloneData(), cloneIndex(), comparator);
}
@Override
- public SortedSource loadSorted(Comparator<BytesRef> comp)
- throws IOException {
- IndexInput indexIn = cloneIndex();
- return new Source(cloneData(), indexIn, comp, indexIn.readLong());
+ public Source getDirectSource() throws IOException {
+ return new DirectSortedSource(cloneData(), cloneIndex(), comparator, type());
}
+
+ }
+ private static final class VarSortedSource extends BytesSortedSourceBase {
+ private final PackedInts.Reader ordToOffsetIndex; // 0-based
+ private final int valueCount;
- private static class Source extends BytesSortedSourceBase {
- private final PackedInts.Reader ordToOffsetIndex; // 0-based
- private final long totBytes;
- private final int valueCount;
-
- public Source(IndexInput datIn, IndexInput idxIn,
- Comparator<BytesRef> comp, long dataLength) throws IOException {
- super(datIn, idxIn, comp, dataLength, ValueType.BYTES_VAR_SORTED);
- totBytes = dataLength;
- ordToOffsetIndex = PackedInts.getReader(idxIn);
- valueCount = ordToOffsetIndex.size();
- closeIndexInput();
- }
-
- @Override
- public int getByValue(BytesRef bytes, BytesRef tmpRef) {
- return binarySearch(bytes, tmpRef, 0, valueCount - 1);
- }
-
- @Override
- public int getValueCount() {
- return valueCount;
- }
+ VarSortedSource(IndexInput datIn, IndexInput idxIn,
+ Comparator<BytesRef> comp) throws IOException {
+ super(datIn, idxIn, comp, idxIn.readLong(), ValueType.BYTES_VAR_SORTED);
+ ordToOffsetIndex = PackedInts.getReader(idxIn);
+ valueCount = ordToOffsetIndex.size()-1; // the last value here is just a dummy value to get the length of the last value
+ closeIndexInput();
+ }
- // ord is 0-based
- @Override
- protected BytesRef deref(int ord, BytesRef bytesRef) {
- final long nextOffset;
- if (ord == valueCount - 1) {
- nextOffset = totBytes;
- } else {
- nextOffset = ordToOffsetIndex.get(1 + ord);
- }
- final long offset = ordToOffsetIndex.get(ord);
- data.fillSlice(bytesRef, offset, (int) (nextOffset - offset));
- return bytesRef;
- }
+ @Override
+ public BytesRef getByOrd(int ord, BytesRef bytesRef) {
+ final long offset = ordToOffsetIndex.get(ord);
+ final long nextOffset = ordToOffsetIndex.get(1 + ord);
+ data.fillSlice(bytesRef, offset, (int) (nextOffset - offset));
+ return bytesRef;
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new VarSortedBytesEnum(source, cloneData(), cloneIndex());
+ public int getValueCount() {
+ return valueCount;
}
+ }
- private static class VarSortedBytesEnum extends ValuesEnum {
- private PackedInts.Reader docToOrdIndex;
- private PackedInts.Reader ordToOffsetIndex;
- private IndexInput idxIn;
- private IndexInput datIn;
- private int valueCount;
- private long totBytes;
- private int docCount;
- private int pos = -1;
- private final long fp;
-
- protected VarSortedBytesEnum(AttributeSource source, IndexInput datIn,
- IndexInput idxIn) throws IOException {
- super(source, ValueType.BYTES_VAR_SORTED);
- totBytes = idxIn.readLong();
- // keep that in memory to prevent lots of disk seeks
- docToOrdIndex = PackedInts.getReader(idxIn);
- ordToOffsetIndex = PackedInts.getReader(idxIn);
- valueCount = ordToOffsetIndex.size();
- docCount = docToOrdIndex.size();
- fp = datIn.getFilePointer();
- this.idxIn = idxIn;
- this.datIn = datIn;
- }
+ private static final class DirectSortedSource extends SortedSource {
+ private final PackedInts.Reader docToOrdIndex;
+ private final PackedInts.RandomAccessReaderIterator ordToOffsetIndex;
+ private final IndexInput datIn;
+ private final long basePointer;
+ private final int valueCount;
+
+ DirectSortedSource(IndexInput datIn, IndexInput idxIn,
+ Comparator<BytesRef> comparator, ValueType type) throws IOException {
+ super(type, comparator);
+ idxIn.readLong();
+ docToOrdIndex = PackedInts.getReader(idxIn); // read the ords in to prevent too many random disk seeks
+ ordToOffsetIndex = PackedInts.getRandomAccessReaderIterator(idxIn);
+ valueCount = ordToOffsetIndex.size()-1; // the last value here is just a dummy value to get the length of the last value
+ basePointer = datIn.getFilePointer();
+ this.datIn = datIn;
+ }
- @Override
- public void close() throws IOException {
- idxIn.close();
- datIn.close();
- }
+ @Override
+ public int ord(int docID) {
+ return (int) docToOrdIndex.get(docID);
+ }
- @Override
- public int advance(int target) throws IOException {
- if (target >= docCount) {
- return pos = NO_MORE_DOCS;
- }
- int ord;
- while ((ord = (int) docToOrdIndex.get(target)) == 0) {
- if (++target >= docCount) {
- return pos = NO_MORE_DOCS;
- }
- }
- final long offset = ordToOffsetIndex.get(--ord);
- final long nextOffset;
- if (ord == valueCount - 1) {
- nextOffset = totBytes;
- } else {
- nextOffset = ordToOffsetIndex.get(1 + ord);
- }
+ @Override
+ public BytesRef getByOrd(int ord, BytesRef bytesRef) {
+ try {
+ final long offset = ordToOffsetIndex.get(ord);
+ final long nextOffset = ordToOffsetIndex.next();
+ datIn.seek(basePointer + offset);
final int length = (int) (nextOffset - offset);
- datIn.seek(fp + offset);
- if (bytesRef.bytes.length < length)
+ if (bytesRef.bytes.length < length) {
bytesRef.grow(length);
+ }
datIn.readBytes(bytesRef.bytes, 0, length);
bytesRef.length = length;
bytesRef.offset = 0;
- return pos = target;
- }
+ return bytesRef;
+ } catch (IOException ex) {
+ throw new IllegalStateException("failed", ex);
- @Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= docCount) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
}
}
-
+
@Override
- public ValueType type() {
- return ValueType.BYTES_VAR_SORTED;
+ public int getValueCount() {
+ return valueCount;
}
+
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Fri Oct 7 09:05:10 2011
@@ -20,18 +20,19 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
+import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase;
+import org.apache.lucene.index.values.DirectSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
import org.apache.lucene.util.packed.PackedInts;
@@ -66,7 +67,7 @@ class VarStraightBytesImpl {
}
// Fills up to but not including this docID
- private void fill(final int docID) {
+ private void fill(final int docID, final long nextAddress) {
if (docID >= docToAddress.length) {
int oldSize = docToAddress.length;
docToAddress = ArrayUtil.grow(docToAddress, 1 + docID);
@@ -74,7 +75,7 @@ class VarStraightBytesImpl {
* RamUsageEstimator.NUM_BYTES_INT);
}
for (int i = lastDocID + 1; i < docID; i++) {
- docToAddress[i] = address;
+ docToAddress[i] = nextAddress;
}
}
@@ -84,7 +85,7 @@ class VarStraightBytesImpl {
if (bytes.length == 0) {
return; // default
}
- fill(docID);
+ fill(docID, address);
docToAddress[docID] = address;
pool.copy(bytes);
address += bytes.length;
@@ -97,15 +98,15 @@ class VarStraightBytesImpl {
datOut = getOrCreateDataOut();
boolean success = false;
try {
- if (state.liveDocs == null && state.reader instanceof Reader) {
+ if (state.liveDocs == null && state.reader instanceof VarStraightReader) {
// bulk merge since we don't have any deletes
- Reader reader = (Reader) state.reader;
+ VarStraightReader reader = (VarStraightReader) state.reader;
final int maxDocs = reader.maxDoc;
if (maxDocs == 0) {
return;
}
if (lastDocID+1 < state.docBase) {
- fill(state.docBase);
+ fill(state.docBase, address);
lastDocID = state.docBase-1;
}
final long numDataBytes;
@@ -147,13 +148,14 @@ class VarStraightBytesImpl {
}
@Override
- protected void mergeDoc(int docID) throws IOException {
+ protected void mergeDoc(int docID, int sourceDoc) throws IOException {
assert merge;
assert lastDocID < docID;
+ currentMergeSource.getBytes(sourceDoc, bytesRef);
if (bytesRef.length == 0) {
return; // default
}
- fill(docID);
+ fill(docID, address);
datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
docToAddress[docID] = address;
address += bytesRef.length;
@@ -186,20 +188,21 @@ class VarStraightBytesImpl {
try {
if (lastDocID == -1) {
idxOut.writeVLong(0);
- final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
+ final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount+1,
PackedInts.bitsRequired(0));
- for (int i = 0; i < docCount; i++) {
+ for (int i = 0; i < docCount+1; i++) {
w.add(0);
}
w.finish();
} else {
- fill(docCount);
+ fill(docCount, address);
idxOut.writeVLong(address);
- final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
+ final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount+1,
PackedInts.bitsRequired(address));
for (int i = 0; i < docCount; i++) {
w.add(docToAddress[i]);
}
+ w.add(address);
w.finish();
}
success = true;
@@ -220,115 +223,59 @@ class VarStraightBytesImpl {
}
}
- public static class Reader extends BytesReaderBase {
+ public static class VarStraightReader extends BytesReaderBase {
private final int maxDoc;
- Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true, context);
+ VarStraightReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context, ValueType.BYTES_VAR_STRAIGHT);
this.maxDoc = maxDoc;
}
@Override
public Source load() throws IOException {
- return new Source(cloneData(), cloneIndex());
+ return new VarStraightSource(cloneData(), cloneIndex());
}
- private class Source extends DerefBytesSourceBase {
-
- public Source(IndexInput datIn, IndexInput idxIn) throws IOException {
- super(datIn, idxIn, idxIn.readVLong(), ValueType.BYTES_VAR_STRAIGHT);
- }
-
- @Override
- public BytesRef getBytes(int docID, BytesRef bytesRef) {
- final long address = addresses.get(docID);
- final int length = docID == maxDoc - 1 ? (int) (totalLengthInBytes - address)
- : (int) (addresses.get(1 + docID) - address);
- return data.fillSlice(bytesRef, address, length);
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return new SourceEnum(attrSource, type(), this, maxDoc()) {
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs) {
- return pos = NO_MORE_DOCS;
- }
- source.getBytes(target, bytesRef);
- return pos = target;
- }
- };
- }
+ @Override
+ public Source getDirectSource()
+ throws IOException {
+ return new DirectVarStraightSource(cloneData(), cloneIndex(), type());
+ }
+ }
+
+ private static final class VarStraightSource extends BytesSourceBase {
+ private final PackedInts.Reader addresses;
+
+ public VarStraightSource(IndexInput datIn, IndexInput idxIn) throws IOException {
+ super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), idxIn.readVLong(),
+ ValueType.BYTES_VAR_STRAIGHT);
+ addresses = PackedInts.getReader(idxIn);
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new VarStraightBytesEnum(source, cloneData(), cloneIndex());
+ public BytesRef getBytes(int docID, BytesRef bytesRef) {
+ final long address = addresses.get(docID);
+ return data.fillSlice(bytesRef, address,
+ (int) (addresses.get(docID + 1) - address));
}
+ }
+
+ public final static class DirectVarStraightSource extends DirectSource {
- private class VarStraightBytesEnum extends ValuesEnum {
- private final PackedInts.ReaderIterator addresses;
- private final IndexInput datIn;
- private final IndexInput idxIn;
- private final long fp;
- private final long totBytes;
- private int pos = -1;
- private long nextAddress;
-
- protected VarStraightBytesEnum(AttributeSource source, IndexInput datIn,
- IndexInput idxIn) throws IOException {
- super(source, ValueType.BYTES_VAR_STRAIGHT);
- totBytes = idxIn.readVLong();
- fp = datIn.getFilePointer();
- addresses = PackedInts.getReaderIterator(idxIn);
- this.datIn = datIn;
- this.idxIn = idxIn;
- nextAddress = addresses.next();
- }
-
- @Override
- public void close() throws IOException {
- datIn.close();
- idxIn.close();
- }
-
- @Override
- public int advance(final int target) throws IOException {
- if (target >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- final long addr = pos+1 == target ? nextAddress : addresses.advance(target);
- if (addr == totBytes) { // empty values at the end
- bytesRef.length = 0;
- bytesRef.offset = 0;
- return pos = target;
- }
- datIn.seek(fp + addr);
- final int size = (int) (target == maxDoc - 1 ? totBytes - addr
- : (nextAddress = addresses.next()) - addr);
- if (bytesRef.bytes.length < size) {
- bytesRef.grow(size);
- }
- bytesRef.length = size;
- datIn.readBytes(bytesRef.bytes, 0, size);
- return pos = target;
- }
-
- @Override
- public int docID() {
- return pos;
- }
+ private final PackedInts.RandomAccessReaderIterator index;
- @Override
- public int nextDoc() throws IOException {
- return advance(pos + 1);
- }
+ DirectVarStraightSource(IndexInput data, IndexInput index, ValueType type)
+ throws IOException {
+ super(data, type);
+ index.readVLong();
+ this.index = PackedInts.getRandomAccessReaderIterator(index);
}
@Override
- public ValueType type() {
- return ValueType.BYTES_VAR_STRAIGHT;
+ protected int position(int docID) throws IOException {
+ final long offset = index.get(docID);
+ data.seek(baseOffset + offset);
+ return (int) (index.next() - offset);
}
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/Writer.java Fri Oct 7 09:05:10 2011
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.Comparator;
import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
@@ -39,7 +40,7 @@ import org.apache.lucene.util.Counter;
* @lucene.experimental
*/
public abstract class Writer extends DocValuesConsumer {
-
+ protected Source currentMergeSource;
/**
* Creates a new {@link Writer}.
*
@@ -99,31 +100,32 @@ public abstract class Writer extends Doc
}
/**
- * Records a value from the given document id. The methods implementation
- * obtains the value for the document id from the last {@link ValuesEnum}
- * set to {@link #setNextEnum(ValuesEnum)}.
+ * Merges a document with the given <code>docID</code>. The methods
+ * implementation obtains the value for the <i>sourceDoc</i> id from the
+ * current {@link Source} set to <i>setNextMergeSource(Source)</i>.
* <p>
* This method is used during merging to provide implementation agnostic
* default merge implementation.
* </p>
* <p>
- * The given document id must be the same document id returned from
- * {@link ValuesEnum#docID()} when this method is called. All documents IDs
- * between the given ID and the previously given ID or <tt>0</tt> if the
- * method is call the first time are filled with default values depending on
- * the {@link Writer} implementation. The given document ID must always be
- * greater than the previous ID or <tt>0</tt> if called the first time.
+ * All documents IDs between the given ID and the previously given ID or
+ * <tt>0</tt> if the method is call the first time are filled with default
+ * values depending on the {@link Writer} implementation. The given document
+ * ID must always be greater than the previous ID or <tt>0</tt> if called the
+ * first time.
*/
- protected abstract void mergeDoc(int docID) throws IOException;
+ protected abstract void mergeDoc(int docID, int sourceDoc) throws IOException;
/**
- * Sets the next {@link ValuesEnum} to consume values from on calls to
- * {@link #mergeDoc(int)}
+ * Sets the next {@link Source} to consume values from on calls to
+ * {@link #mergeDoc(int, int)}
*
- * @param valuesEnum
- * the next {@link ValuesEnum}, this must not be null
+ * @param mergeSource
+ * the next {@link Source}, this must not be null
*/
- protected abstract void setNextEnum(ValuesEnum valuesEnum);
+ protected void setNextMergeSource(Source mergeSource) {
+ currentMergeSource = mergeSource;
+ }
/**
* Finish writing and close any files and resources used by this Writer.
@@ -141,34 +143,20 @@ public abstract class Writer extends Doc
// simply override this and decide if they want to merge
// segments using this generic implementation or if a bulk merge is possible
// / feasible.
- final ValuesEnum valEnum = state.reader.getEnum();
- assert valEnum != null;
- try {
- setNextEnum(valEnum); // set the current enum we are working on - the
- // impl. will get the correct reference for the type
- // it supports
- int docID = state.docBase;
- final Bits liveDocs = state.liveDocs;
- final int docCount = state.docCount;
- int currentDocId;
- if ((currentDocId = valEnum.advance(0)) != ValuesEnum.NO_MORE_DOCS) {
- for (int i = 0; i < docCount; i++) {
- if (liveDocs == null || liveDocs.get(i)) {
- if (currentDocId < i) {
- if ((currentDocId = valEnum.advance(i)) == ValuesEnum.NO_MORE_DOCS) {
- break; // advance can jump over default values
- }
- }
- if (currentDocId == i) { // we are on the doc to merge
- mergeDoc(docID);
- }
- ++docID;
- }
- }
+ final Source source = state.reader.getDirectSource();
+ assert source != null;
+ setNextMergeSource(source); // set the current enum we are working on - the
+ // impl. will get the correct reference for the type
+ // it supports
+ int docID = state.docBase;
+ final Bits liveDocs = state.liveDocs;
+ final int docCount = state.docCount;
+ for (int i = 0; i < docCount; i++) {
+ if (liveDocs == null || liveDocs.get(i)) {
+ mergeDoc(docID++, i);
}
- } finally {
- valEnum.close();
}
+
}
/**
@@ -182,11 +170,6 @@ public abstract class Writer extends Doc
* the file name id used to create files within the writer.
* @param directory
* the {@link Directory} to create the files from.
- * @param comp
- * a {@link BytesRef} comparator used for {@link Bytes} variants. If
- * <code>null</code>
- * {@link BytesRef#getUTF8SortedAsUnicodeComparator()} is used as the
- * default.
* @param bytesUsed
* a byte-usage tracking reference
* @return a new {@link Writer} instance for the given {@link ValueType}
@@ -205,28 +188,27 @@ public abstract class Writer extends Doc
case VAR_INTS:
return Ints.getWriter(directory, id, bytesUsed, type, context);
case FLOAT_32:
- return Floats.getWriter(directory, id, 4, bytesUsed, context);
+ return Floats.getWriter(directory, id, bytesUsed, context, type);
case FLOAT_64:
- return Floats.getWriter(directory, id, 8, bytesUsed, context);
+ return Floats.getWriter(directory, id, bytesUsed, context, type);
case BYTES_FIXED_STRAIGHT:
- return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, true,
+ return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, comp,
bytesUsed, context);
case BYTES_FIXED_DEREF:
- return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, true,
+ return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, comp,
bytesUsed, context);
case BYTES_FIXED_SORTED:
- return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, true,
+ return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, true, comp,
bytesUsed, context);
case BYTES_VAR_STRAIGHT:
- return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, false,
+ return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, comp,
bytesUsed, context);
case BYTES_VAR_DEREF:
- return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, false,
+ return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, comp,
bytesUsed, context);
case BYTES_VAR_SORTED:
- return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, false,
+ return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, false, comp,
bytesUsed, context);
-
default:
throw new IllegalArgumentException("Unknown Values: " + type);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java Fri Oct 7 09:05:10 2011
@@ -85,6 +85,14 @@ public class PackedInts {
long advance(int ord) throws IOException;
}
+ public static interface RandomAccessReaderIterator extends ReaderIterator {
+ /**
+ * @param index the position of the wanted value.
+ * @return the value at the stated index.
+ */
+ long get(int index) throws IOException;
+ }
+
/**
* A packed integer array that can be modified.
* @lucene.internal
@@ -195,6 +203,17 @@ public class PackedInts {
* @lucene.internal
*/
public static ReaderIterator getReaderIterator(IndexInput in) throws IOException {
+ return getRandomAccessReaderIterator(in);
+ }
+
+ /**
+ * Retrieve PackedInts as a {@link RandomAccessReaderIterator}
+ * @param in positioned at the beginning of a stored packed int structure.
+ * @return an iterator to access the values
+ * @throws IOException if the structure could not be retrieved.
+ * @lucene.internal
+ */
+ public static RandomAccessReaderIterator getRandomAccessReaderIterator(IndexInput in) throws IOException {
CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java Fri Oct 7 09:05:10 2011
@@ -21,13 +21,15 @@ import org.apache.lucene.store.IndexInpu
import java.io.IOException;
-final class PackedReaderIterator implements PackedInts.ReaderIterator {
+final class PackedReaderIterator implements PackedInts.RandomAccessReaderIterator {
private long pending;
private int pendingBitsLeft;
private final IndexInput in;
private final int bitsPerValue;
private final int valueCount;
private int position = -1;
+ private long currentValue;
+ private final long startPointer;
// masks[n-1] masks for bottom n bits
private final long[] masks;
@@ -39,6 +41,7 @@ final class PackedReaderIterator impleme
this.bitsPerValue = bitsPerValue;
this.in = in;
+ startPointer = in.getFilePointer();
masks = new long[bitsPerValue];
long v = 1;
@@ -76,7 +79,7 @@ final class PackedReaderIterator impleme
}
++position;
- return result;
+ return currentValue = result;
}
public void close() throws IOException {
@@ -106,6 +109,26 @@ final class PackedReaderIterator impleme
pendingBitsLeft = 64 - (int)(skip % 64);
}
position = ord-1;
- return next();
+ return currentValue = next();
+ }
+
+
+ @Override
+ public long get(int index) throws IOException {
+ assert index < valueCount : "ord must be less than valueCount";
+ if (index < position) {
+ pendingBitsLeft = 0;
+ final long bitsToSkip = (((long) bitsPerValue) * (long) index);
+ final long skip = bitsToSkip - pendingBitsLeft;
+ final long closestByte = (skip >> 6) << 3;
+ in.seek(startPointer + closestByte);
+ pending = in.readLong();
+ pendingBitsLeft = 64 - (int) (skip % 64);
+ position = index - 1;
+ return currentValue = next();
+ } else if (index == position) {
+ return currentValue;
+ }
+ return advance(index);
}
}
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1179970&r1=1179969&r2=1179970&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Fri Oct 7 09:05:10 2011
@@ -176,8 +176,8 @@ public class RandomIndexWriter implement
IndexDocValuesField docValuesField = new IndexDocValuesField(name);
switch (type) {
case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
case BYTES_FIXED_STRAIGHT:
+ case BYTES_FIXED_SORTED:
final String randomUnicodeString = _TestUtil.randomUnicodeString(random, fixedBytesLength);
BytesRef fixedRef = new BytesRef(randomUnicodeString);
if (fixedRef.length > fixedBytesLength) {
@@ -189,8 +189,8 @@ public class RandomIndexWriter implement
docValuesField.setBytes(fixedRef, type);
break;
case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
case BYTES_VAR_STRAIGHT:
+ case BYTES_VAR_SORTED:
BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200));
docValuesField.setBytes(ref, type);
break;