You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/10/10 21:58:32 UTC
svn commit: r1181188 [4/8] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/modules/grouping/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/MultiIndexDocValues.java Mon Oct 10 19:58:24 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
@@ -28,6 +27,7 @@ import org.apache.lucene.util.ReaderUtil
* {@link IndexDocValues}
*
* @lucene.experimental
+ * @lucene.internal
*/
public class MultiIndexDocValues extends IndexDocValues {
@@ -46,6 +46,8 @@ public class MultiIndexDocValues extends
private DocValuesIndex[] docValuesIdx;
private int[] starts;
+ private ValueType type;
+ private int valueSize;
public MultiIndexDocValues() {
starts = new int[0];
@@ -57,111 +59,56 @@ public class MultiIndexDocValues extends
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new MultiValuesEnum(docValuesIdx, starts);
- }
-
- @Override
public Source load() throws IOException {
- return new MultiSource(docValuesIdx, starts);
+ return new MultiSource(docValuesIdx, starts, false);
}
public IndexDocValues reset(DocValuesIndex[] docValuesIdx) {
- int[] start = new int[docValuesIdx.length];
+ final int[] start = new int[docValuesIdx.length];
+ TypePromoter promoter = TypePromoter.getIdentityPromoter();
for (int i = 0; i < docValuesIdx.length; i++) {
start[i] = docValuesIdx[i].start;
+ if (!(docValuesIdx[i].docValues instanceof EmptyDocValues)) {
+ // only promote if not a dummy
+ final TypePromoter incomingPromoter = TypePromoter.create(
+ docValuesIdx[i].docValues.type(),
+ docValuesIdx[i].docValues.getValueSize());
+ promoter = promoter.promote(incomingPromoter);
+ if (promoter == null) {
+ throw new IllegalStateException("Can not promote " + incomingPromoter);
+ }
+ }
}
+ this.type = promoter.type();
+ this.valueSize = promoter.getValueSize();
this.starts = start;
this.docValuesIdx = docValuesIdx;
return this;
}
- public static class DummyDocValues extends IndexDocValues {
+ public static class EmptyDocValues extends IndexDocValues {
final int maxDoc;
- final Source emptySoruce;
+ final Source emptySource;
- public DummyDocValues(int maxDoc, ValueType type) {
+ public EmptyDocValues(int maxDoc, ValueType type) {
this.maxDoc = maxDoc;
- this.emptySoruce = new EmptySource(type);
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return emptySoruce.getEnum(attrSource);
+ this.emptySource = new EmptySource(type);
}
@Override
public Source load() throws IOException {
- return emptySoruce;
+ return emptySource;
}
@Override
public ValueType type() {
- return emptySoruce.type();
- }
- }
-
- private static class MultiValuesEnum extends ValuesEnum {
- private DocValuesIndex[] docValuesIdx;
- private final int maxDoc;
- private int currentStart;
- private int currentMax;
- private int currentDoc = -1;
- private ValuesEnum currentEnum;
- private final int[] starts;
-
- public MultiValuesEnum(DocValuesIndex[] docValuesIdx, int[] starts)
- throws IOException {
- super(docValuesIdx[0].docValues.type());
- this.docValuesIdx = docValuesIdx;
- final DocValuesIndex last = docValuesIdx[docValuesIdx.length - 1];
- maxDoc = last.start + last.length;
- final DocValuesIndex idx = docValuesIdx[0];
- currentEnum = idx.docValues.getEnum(this.attributes());
- currentEnum.copyFrom(this);
- intsRef = currentEnum.intsRef;
- currentMax = idx.length;
- currentStart = 0;
- this.starts = starts;
- }
-
- @Override
- public void close() throws IOException {
- currentEnum.close();
- }
-
- @Override
- public int advance(int target) throws IOException {
- assert target > currentDoc : "target " + target
- + " must be > than the current doc " + currentDoc;
- int relativeDoc = target - currentStart;
- do {
- if (target >= maxDoc) {// we are beyond max doc
- return currentDoc = NO_MORE_DOCS;
- }
- if (target >= currentMax) {
- final int idx = ReaderUtil.subIndex(target, starts);
- currentEnum.close();
- currentEnum = docValuesIdx[idx].docValues.getEnum();
- currentEnum.copyFrom(this);
- currentStart = docValuesIdx[idx].start;
- currentMax = currentStart + docValuesIdx[idx].length;
- relativeDoc = target - currentStart;
- }
- target = currentMax; // make sure that we advance to the next enum if the current is exhausted
-
- } while ((relativeDoc = currentEnum.advance(relativeDoc)) == NO_MORE_DOCS);
- return currentDoc = currentStart + relativeDoc;
+ return emptySource.type();
}
- @Override
- public int docID() {
- return currentDoc;
- }
@Override
- public int nextDoc() throws IOException {
- return advance(currentDoc + 1);
+ public Source getDirectSource() throws IOException {
+ return emptySource;
}
}
@@ -171,12 +118,14 @@ public class MultiIndexDocValues extends
private Source current;
private final int[] starts;
private final DocValuesIndex[] docValuesIdx;
+ private boolean direct;
- public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts) {
+ public MultiSource(DocValuesIndex[] docValuesIdx, int[] starts, boolean direct) {
+ super(docValuesIdx[0].docValues.type());
this.docValuesIdx = docValuesIdx;
this.starts = starts;
assert docValuesIdx.length != 0;
-
+ this.direct = direct;
}
public long getInt(int docID) {
@@ -193,7 +142,11 @@ public class MultiIndexDocValues extends
+ " for doc id: " + docID + " slices : " + Arrays.toString(starts);
assert docValuesIdx[idx] != null;
try {
- current = docValuesIdx[idx].docValues.getSource();
+ if (direct) {
+ current = docValuesIdx[idx].docValues.getDirectSource();
+ } else {
+ current = docValuesIdx[idx].docValues.getSource();
+ }
} catch (IOException e) {
throw new RuntimeException("load failed", e); // TODO how should we
// handle this
@@ -214,24 +167,12 @@ public class MultiIndexDocValues extends
final int doc = ensureSource(docID);
return current.getBytes(doc, bytesRef);
}
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- throw new UnsupportedOperationException(); // TODO
- }
-
- @Override
- public ValueType type() {
- return docValuesIdx[0].docValues.type();
- }
-
}
private static class EmptySource extends Source {
- private final ValueType type;
public EmptySource(ValueType type) {
- this.type = type;
+ super(type);
}
@Override
@@ -250,20 +191,20 @@ public class MultiIndexDocValues extends
public long getInt(int docID) {
return 0;
}
+ }
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return ValuesEnum.emptyEnum(type);
- }
+ @Override
+ public ValueType type() {
+ return type;
+ }
- @Override
- public ValueType type() {
- return type;
- }
+ @Override
+ public int getValueSize() {
+ return valueSize;
}
@Override
- public ValueType type() {
- return this.docValuesIdx[0].docValues.type();
+ public Source getDirectSource() throws IOException {
+ return new MultiSource(docValuesIdx, starts, true);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java Mon Oct 10 19:58:24 2011
@@ -21,18 +21,15 @@ import java.io.IOException;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.values.FixedStraightBytesImpl.FixedBytesWriterBase;
import org.apache.lucene.index.values.IndexDocValues.Source;
-import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
import org.apache.lucene.index.values.IndexDocValuesArray.LongValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.packed.PackedInts;
/**
@@ -51,7 +48,6 @@ class PackedIntValues {
static class PackedIntsWriter extends FixedBytesWriterBase {
- private LongsRef intsRef;
private long minValue;
private long maxValue;
private boolean started;
@@ -114,10 +110,10 @@ class PackedIntValues {
}
@Override
- protected void mergeDoc(int docID) throws IOException {
+ protected void mergeDoc(int docID, int sourceDoc) throws IOException {
assert docID > lastDocId : "docID: " + docID
+ " must be greater than the last added doc id: " + lastDocId;
- add(docID, intsRef.get());
+ add(docID, currentMergeSource.getInt(sourceDoc));
}
private void writePackedInts(IndexOutput datOut, int docCount) throws IOException {
@@ -139,12 +135,6 @@ class PackedIntValues {
w.add(defaultValue);
}
w.finish();
- w.finish();
- }
-
- @Override
- protected void setNextEnum(ValuesEnum valuesEnum) {
- intsRef = valuesEnum.getInt();
}
@Override
@@ -215,30 +205,17 @@ class PackedIntValues {
datIn.close();
}
- @Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- final IndexInput input = (IndexInput) datIn.clone();
- boolean success = false;
- try {
- final ValuesEnum inst;
- if (values == null) {
- inst = new PackedIntsEnumImpl(source, input);
- } else {
- inst = values.getDirectEnum(source, input, numDocs);
- }
- success = true;
- return inst;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(input);
- }
- }
- }
@Override
public ValueType type() {
return ValueType.VAR_INTS;
}
+
+
+ @Override
+ public Source getDirectSource() throws IOException {
+ return values != null ? new FixedStraightBytesImpl.DirectFixedStraightSource((IndexInput) datIn.clone(), 8, ValueType.FIXED_INTS_64) : new DirectPackedIntsSource((IndexInput) datIn.clone());
+ }
}
@@ -248,7 +225,7 @@ class PackedIntValues {
private final PackedInts.Reader values;
public PackedIntsSource(IndexInput dataIn) throws IOException {
-
+ super(ValueType.VAR_INTS);
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
values = PackedInts.getReader(dataIn);
@@ -263,72 +240,41 @@ class PackedIntValues {
final long value = values.get(docID);
return value == defaultValue ? 0 : minValue + value;
}
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return new SourceEnum(attrSource, type(), this, values.size()) {
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs)
- return pos = NO_MORE_DOCS;
- intsRef.ints[intsRef.offset] = source.getInt(target);
- return pos = target;
- }
- };
- }
-
- @Override
- public ValueType type() {
- return ValueType.VAR_INTS;
- }
}
- private static final class PackedIntsEnumImpl extends ValuesEnum {
- private final PackedInts.ReaderIterator ints;
+ private static final class DirectPackedIntsSource extends Source {
+ private final PackedInts.RandomAccessReaderIterator ints;
private long minValue;
- private final IndexInput dataIn;
private final long defaultValue;
- private final int maxDoc;
- private int pos = -1;
- private PackedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
+ private DirectPackedIntsSource(IndexInput dataIn)
throws IOException {
- super(source, ValueType.VAR_INTS);
- intsRef.offset = 0;
- this.dataIn = dataIn;
+ super(ValueType.VAR_INTS);
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
- this.ints = PackedInts.getReaderIterator(dataIn);
- maxDoc = ints.size();
+ this.ints = PackedInts.getRandomAccessReaderIterator(dataIn);
}
@Override
- public void close() throws IOException {
- ints.close();
- dataIn.close();
+ public double getFloat(int docID) {
+ return getInt(docID);
}
@Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- final long val = ints.advance(target);
- intsRef.ints[intsRef.offset] = val == defaultValue ? 0 : minValue + val;
- return pos = target;
+ public BytesRef getBytes(int docID, BytesRef ref) {
+ ref.grow(8);
+ ref.copy(getInt(docID));
+ return ref;
}
@Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= maxDoc) {
- return pos = NO_MORE_DOCS;
+ public long getInt(int docID) {
+ try {
+ final long val = ints.get(docID);
+ return val == defaultValue ? 0 : minValue + val;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
- return advance(pos + 1);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/SourceCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/SourceCache.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/SourceCache.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/SourceCache.java Mon Oct 10 19:58:24 2011
@@ -18,36 +18,29 @@ package org.apache.lucene.index.values;
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.values.IndexDocValues.SortedSource;
import org.apache.lucene.index.values.IndexDocValues.Source;
-import org.apache.lucene.util.BytesRef;
/**
- * Abstract base class for {@link IndexDocValues} {@link Source} /
- * {@link SortedSource} cache.
+ * Abstract base class for {@link IndexDocValues} {@link Source} cache.
* <p>
- * {@link Source} and {@link SortedSource} instances loaded via
- * {@link IndexDocValues#load()} and {@link IndexDocValues#loadSorted(Comparator)} are
- * entirely memory resident and need to be maintained by the caller. Each call
- * to {@link IndexDocValues#load()} or {@link IndexDocValues#loadSorted(Comparator)} will
- * cause an entire reload of the underlying data. Source and
- * {@link SortedSource} instances obtained from {@link IndexDocValues#getSource()}
- * and {@link IndexDocValues#getSource()} respectively are maintained by a
- * {@link SourceCache} that is closed ({@link #close(IndexDocValues)}) once the
- * {@link IndexReader} that created the {@link IndexDocValues} instance is closed.
+ * {@link Source} instances loaded via {@link IndexDocValues#load()} are entirely memory resident
+ * and need to be maintained by the caller. Each call to
+ * {@link IndexDocValues#load()} will cause an entire reload of
+ * the underlying data. Source instances obtained from
+ * {@link IndexDocValues#getSource()} and {@link IndexDocValues#getSource()}
+ * respectively are maintained by a {@link SourceCache} that is closed (
+ * {@link #close(IndexDocValues)}) once the {@link IndexReader} that created the
+ * {@link IndexDocValues} instance is closed.
* <p>
- * Unless {@link Source} and {@link SortedSource} instances are managed by
- * another entity it is recommended to use the cached variants to obtain a
- * source instance.
+ * Unless {@link Source} instances are managed by another entity it is
+ * recommended to use the cached variants to obtain a source instance.
* <p>
* Implementation of this API must be thread-safe.
*
* @see IndexDocValues#setCache(SourceCache)
* @see IndexDocValues#getSource()
- * @see IndexDocValues#getSortedSorted(Comparator)
*
* @lucene.experimental
*/
@@ -63,17 +56,7 @@ public abstract class SourceCache {
public abstract Source load(IndexDocValues values) throws IOException;
/**
- * Atomically loads a {@link SortedSource} into the cache from the given
- * {@link IndexDocValues} and returns it iff no other {@link SortedSource} has
- * already been cached. Otherwise the cached source is returned.
- * <p>
- * This method will not return <code>null</code>
- */
- public abstract SortedSource loadSorted(IndexDocValues values,
- Comparator<BytesRef> comp) throws IOException;
-
- /**
- * Atomically invalidates the cached {@link Source} and {@link SortedSource}
+ * Atomically invalidates the cached {@link Source}
* instances if any and empties the cache.
*/
public abstract void invalidate(IndexDocValues values);
@@ -87,14 +70,13 @@ public abstract class SourceCache {
/**
* Simple per {@link IndexDocValues} instance cache implementation that holds a
- * {@link Source} and {@link SortedSource} reference as a member variable.
+ * {@link Source} a member variable.
* <p>
* If a {@link DirectSourceCache} instance is closed or invalidated the cached
* reference are simply set to <code>null</code>
*/
public static final class DirectSourceCache extends SourceCache {
private Source ref;
- private SortedSource sortedRef;
public synchronized Source load(IndexDocValues values) throws IOException {
if (ref == null) {
@@ -103,17 +85,8 @@ public abstract class SourceCache {
return ref;
}
- public synchronized SortedSource loadSorted(IndexDocValues values,
- Comparator<BytesRef> comp) throws IOException {
- if (sortedRef == null) {
- sortedRef = values.loadSorted(comp);
- }
- return sortedRef;
- }
-
public synchronized void invalidate(IndexDocValues values) {
ref = null;
- sortedRef = null;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/ValueType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/ValueType.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/ValueType.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/ValueType.java Mon Oct 10 19:58:24 2011
@@ -43,9 +43,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
VAR_INTS,
@@ -56,9 +55,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_8,
@@ -69,9 +67,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_16,
@@ -82,9 +79,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_32,
@@ -95,9 +91,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
- * documents without an explicit value will use <tt>0</tt> instead. In turn,
- * {@link ValuesEnum} instances will not skip documents without an explicit
- * value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FIXED_INTS_64,
@@ -110,9 +105,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0.0f</tt> as the default value without any
* distinction between provided <tt>0.0f</tt> values during indexing. All
- * documents without an explicit value will use <tt>0.0f</tt> instead. In
- * turn, {@link ValuesEnum} instances will not skip documents without an
- * explicit value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0.0f</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FLOAT_32,
@@ -126,9 +120,8 @@ public enum ValueType {
* <p>
* NOTE: this type uses <tt>0.0d</tt> as the default value without any
* distinction between provided <tt>0.0d</tt> values during indexing. All
- * documents without an explicit value will use <tt>0.0d</tt> instead. In
- * turn, {@link ValuesEnum} instances will not skip documents without an
- * explicit value assigned. Custom default values must be assigned explicitly.
+ * documents without an explicit value will use <tt>0.0d</tt> instead.
+ * Custom default values must be assigned explicitly.
* </p>
*/
FLOAT_64,
@@ -143,9 +136,7 @@ public enum ValueType {
* NOTE: this type uses <tt>0 byte</tt> filled byte[] based on the length of the first seen
* value as the default value without any distinction between explicitly
* provided values during indexing. All documents without an explicit value
- * will use the default instead. In turn, {@link ValuesEnum} instances will
- * not skip documents without an explicit value assigned. Custom default
- * values must be assigned explicitly.
+ * will use the default instead.Custom default values must be assigned explicitly.
* </p>
*/
BYTES_FIXED_STRAIGHT,
@@ -159,34 +150,12 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without and
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
- * explicitly.
+ * reference. Custom default values must be assigned explicitly.
* </p>
*/
BYTES_FIXED_DEREF,
/**
- * A fixed length pre-sorted byte[] variant. Fields with this type only
- * store distinct byte values and store an additional offset pointer per
- * document to dereference the shared byte[]. The stored
- * byte[] is presorted, by default by unsigned byte order,
- * and allows access via document id, ordinal and by-value.
- * Use this type if your documents may share the same byte[].
- * <p>
- * NOTE: Fields of this type will not store values for documents without and
- * explicitly provided value. If a documents value is accessed while no
- * explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
- * explicitly.
- * </p>
- *
- * @see SortedSource
- */
- BYTES_FIXED_SORTED,
-
- /**
* Variable length straight stored byte[] variant. All bytes are
* stored sequentially for compactness. Usage of this type via the
* disk-resident API might yield performance degradation since no additional
@@ -195,9 +164,7 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without an
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * byte[] reference. In contrast to dereferenced variants, {@link ValuesEnum}
- * instances will <b>not</b> skip over documents without an explicit value
- * assigned. Custom default values must be assigned explicitly.
+ * byte[] reference. Custom default values must be assigned explicitly.
* </p>
*/
BYTES_VAR_STRAIGHT,
@@ -210,13 +177,12 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without and
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
- * explicitly.
+ * reference. Custom default values must be assigned explicitly.
* </p>
*/
BYTES_VAR_DEREF,
+
/**
* A variable length pre-sorted byte[] variant. Just like
* {@link #BYTES_FIXED_SORTED}, but allowing each
@@ -225,12 +191,30 @@ public enum ValueType {
* NOTE: Fields of this type will not store values for documents without and
* explicitly provided value. If a documents value is accessed while no
* explicit value is stored the returned {@link BytesRef} will be a 0-length
- * reference. In turn, {@link ValuesEnum} instances will skip over documents
- * without an explicit value assigned. Custom default values must be assigned
+ * reference.Custom default values must be assigned explicitly.
+ * </p>
+ *
+ * @see SortedSource
+ */
+ BYTES_VAR_SORTED,
+
+ /**
+ * A fixed length pre-sorted byte[] variant. Fields with this type only
+ * store distinct byte values and store an additional offset pointer per
+ * document to dereference the shared byte[]. The stored
+ * byte[] is presorted, by default by unsigned byte order,
+ * and allows access via document id, ordinal and by-value.
+ * Use this type if your documents may share the same byte[].
+ * <p>
+ * NOTE: Fields of this type will not store values for documents without and
+ * explicitly provided value. If a documents value is accessed while no
+ * explicit value is stored the returned {@link BytesRef} will be a 0-length
+ * reference. Custom default values must be assigned
* explicitly.
* </p>
*
* @see SortedSource
*/
- BYTES_VAR_SORTED
+ BYTES_FIXED_SORTED
+
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Mon Oct 10 19:58:24 2011
@@ -20,16 +20,17 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesEnumBase;
+import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
+import org.apache.lucene.index.values.DirectSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.PagedBytes;
+import org.apache.lucene.util.packed.PackedInts;
// Stores variable-length byte[] by deref, ie when two docs
// have the same value, they store only 1 byte[] and both
@@ -57,6 +58,7 @@ class VarDerefBytesImpl {
public Writer(Directory dir, String id, Counter bytesUsed, IOContext context)
throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
+ size = 0;
}
@Override
@@ -68,88 +70,82 @@ class VarDerefBytesImpl {
// some last docs that we didn't see
@Override
public void finishInternal(int docCount) throws IOException {
+ fillDefault(docCount);
final int size = hash.size();
- final long[] addresses = new long[size+1];
+ final long[] addresses = new long[size];
final IndexOutput datOut = getOrCreateDataOut();
- int addr = 1;
+ int addr = 0;
final BytesRef bytesRef = new BytesRef();
for (int i = 0; i < size; i++) {
hash.get(i, bytesRef);
- addresses[i+1] = addr;
+ addresses[i] = addr;
addr += writePrefixLength(datOut, bytesRef) + bytesRef.length;
datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
final IndexOutput idxOut = getOrCreateIndexOut();
// write the max address to read directly on source load
- idxOut.writeLong(addr - 1);
- writeIndex(idxOut, docCount, addresses[size], addresses, docToEntry);
+ idxOut.writeLong(addr);
+ writeIndex(idxOut, docCount, addresses[addresses.length-1], addresses, docToEntry);
}
}
- public static class Reader extends BytesReaderBase {
+ public static class VarDerefReader extends BytesReaderBase {
private final long totalBytes;
- Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true, context);
+ VarDerefReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context, ValueType.BYTES_VAR_DEREF);
totalBytes = idxIn.readLong();
}
@Override
public Source load() throws IOException {
- return new Source(cloneData(), cloneIndex(), totalBytes);
+ return new VarDerefSource(cloneData(), cloneIndex(), totalBytes);
}
+
+ @Override
+ public Source getDirectSource()
+ throws IOException {
+ return new DirectVarDerefSource(cloneData(), cloneIndex(), type());
+ }
+ }
+
+ final static class VarDerefSource extends BytesSourceBase {
+ private final PackedInts.Reader addresses;
- private final static class Source extends DerefBytesSourceBase {
-
- public Source(IndexInput datIn, IndexInput idxIn, long totalBytes)
- throws IOException {
- super(datIn, idxIn, totalBytes, ValueType.BYTES_VAR_DEREF);
- }
-
- @Override
- public BytesRef getBytes(int docID, BytesRef bytesRef) {
- long address = addresses.get(docID);
- bytesRef.length = 0;
- return address == 0 ? bytesRef : data.fillSliceWithPrefix(bytesRef,
- --address);
- }
+ public VarDerefSource(IndexInput datIn, IndexInput idxIn, long totalBytes)
+ throws IOException {
+ super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), totalBytes,
+ ValueType.BYTES_VAR_DEREF);
+ addresses = PackedInts.getReader(idxIn);
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new VarDerefBytesEnum(source, cloneData(), cloneIndex());
+ public BytesRef getBytes(int docID, BytesRef bytesRef) {
+ return data.fillSliceWithPrefix(bytesRef,
+ addresses.get(docID));
}
+ }
- final static class VarDerefBytesEnum extends DerefBytesEnumBase {
-
- public VarDerefBytesEnum(AttributeSource source, IndexInput datIn,
- IndexInput idxIn) throws IOException {
- super(source, datIn, idxIn, -1, ValueType.BYTES_VAR_DEREF);
- }
+
+ final static class DirectVarDerefSource extends DirectSource {
+ private final PackedInts.RandomAccessReaderIterator index;
- @Override
- protected void fill(long address, BytesRef ref) throws IOException {
- datIn.seek(fp + --address);
- final byte sizeByte = datIn.readByte();
- final int size;
- if ((sizeByte & 128) == 0) {
- // length is 1 byte
- size = sizeByte;
- } else {
- size = ((sizeByte & 0x7f) << 8) | ((datIn.readByte() & 0xff));
- }
- if (ref.bytes.length < size) {
- ref.grow(size);
- }
- ref.length = size;
- ref.offset = 0;
- datIn.readBytes(ref.bytes, 0, size);
- }
+ DirectVarDerefSource(IndexInput data, IndexInput index, ValueType type)
+ throws IOException {
+ super(data, type);
+ this.index = PackedInts.getRandomAccessReaderIterator(index);
}
-
+
@Override
- public ValueType type() {
- return ValueType.BYTES_VAR_DEREF;
+ protected int position(int docID) throws IOException {
+ data.seek(baseOffset + index.get(docID));
+ final byte sizeByte = data.readByte();
+ if ((sizeByte & 128) == 0) {
+ // length is 1 byte
+ return sizeByte;
+ } else {
+ return ((sizeByte & 0x7f) << 8) | ((data.readByte() & 0xff));
+ }
}
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Mon Oct 10 19:58:24 2011
@@ -23,11 +23,11 @@ import java.util.Comparator;
import org.apache.lucene.index.values.Bytes.BytesSortedSourceBase;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
+import org.apache.lucene.index.values.IndexDocValues.SortedSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.packed.PackedInts;
@@ -39,7 +39,7 @@ import org.apache.lucene.util.packed.Pac
/**
* @lucene.experimental
*/
-class VarSortedBytesImpl {
+final class VarSortedBytesImpl {
static final String CODEC_NAME = "VarDerefBytes";
static final int VERSION_START = 0;
@@ -52,8 +52,9 @@ class VarSortedBytesImpl {
Counter bytesUsed, IOContext context) throws IOException {
super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context);
this.comp = comp;
+ size = 0;
}
-
+
@Override
protected void checkSize(BytesRef bytes) {
// allow var bytes sizes
@@ -63,11 +64,11 @@ class VarSortedBytesImpl {
// some last docs that we didn't see
@Override
public void finishInternal(int docCount) throws IOException {
+ fillDefault(docCount);
final int count = hash.size();
final IndexOutput datOut = getOrCreateDataOut();
long offset = 0;
- long lastOffset = 0;
- final int[] index = new int[count+1];
+ final int[] index = new int[count];
final long[] offsets = new long[count];
final int[] sortedEntries = hash.sort(comp);
// first dump bytes data, recording index & offset as
@@ -75,173 +76,125 @@ class VarSortedBytesImpl {
for (int i = 0; i < count; i++) {
final int e = sortedEntries[i];
offsets[i] = offset;
- index[e+1] = 1 + i;
+ index[e] = i;
final BytesRef bytes = hash.get(e, new BytesRef());
// TODO: we could prefix code...
datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
- lastOffset = offset;
offset += bytes.length;
}
-
final IndexOutput idxOut = getOrCreateIndexOut();
// total bytes of data
idxOut.writeLong(offset);
- // write index -- first doc -> 1+ord
+ // write index
writeIndex(idxOut, docCount, count, index, docToEntry);
// next ord (0-based) -> offset
- PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count,
- PackedInts.bitsRequired(lastOffset));
+ PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count+1,
+ PackedInts.bitsRequired(offset));
for (int i = 0; i < count; i++) {
offsetWriter.add(offsets[i]);
}
+ offsetWriter.add(offset);
offsetWriter.finish();
}
}
public static class Reader extends BytesReaderBase {
- private final Comparator<BytesRef> defaultComp;
+ private final Comparator<BytesRef> comparator;
Reader(Directory dir, String id, int maxDoc,
- Comparator<BytesRef> comparator, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true, context);
- this.defaultComp = comparator;
+ IOContext context, ValueType type, Comparator<BytesRef> comparator)
+ throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context, type);
+ this.comparator = comparator;
}
@Override
public org.apache.lucene.index.values.IndexDocValues.Source load()
throws IOException {
- return loadSorted(defaultComp);
+ return new VarSortedSource(cloneData(), cloneIndex(), comparator);
}
@Override
- public SortedSource loadSorted(Comparator<BytesRef> comp)
- throws IOException {
- IndexInput indexIn = cloneIndex();
- return new Source(cloneData(), indexIn, comp, indexIn.readLong());
+ public Source getDirectSource() throws IOException {
+ return new DirectSortedSource(cloneData(), cloneIndex(), comparator, type());
}
+
+ }
+ private static final class VarSortedSource extends BytesSortedSourceBase {
+ private final PackedInts.Reader ordToOffsetIndex; // 0-based
+ private final int valueCount;
- private static class Source extends BytesSortedSourceBase {
- private final PackedInts.Reader ordToOffsetIndex; // 0-based
- private final long totBytes;
- private final int valueCount;
-
- public Source(IndexInput datIn, IndexInput idxIn,
- Comparator<BytesRef> comp, long dataLength) throws IOException {
- super(datIn, idxIn, comp, dataLength, ValueType.BYTES_VAR_SORTED);
- totBytes = dataLength;
- ordToOffsetIndex = PackedInts.getReader(idxIn);
- valueCount = ordToOffsetIndex.size();
- closeIndexInput();
- }
-
- @Override
- public int getByValue(BytesRef bytes, BytesRef tmpRef) {
- return binarySearch(bytes, tmpRef, 0, valueCount - 1);
- }
-
- @Override
- public int getValueCount() {
- return valueCount;
- }
+ VarSortedSource(IndexInput datIn, IndexInput idxIn,
+ Comparator<BytesRef> comp) throws IOException {
+ super(datIn, idxIn, comp, idxIn.readLong(), ValueType.BYTES_VAR_SORTED);
+ ordToOffsetIndex = PackedInts.getReader(idxIn);
+ valueCount = ordToOffsetIndex.size()-1; // the last value here is just a dummy value to get the length of the last value
+ closeIndexInput();
+ }
- // ord is 0-based
- @Override
- protected BytesRef deref(int ord, BytesRef bytesRef) {
- final long nextOffset;
- if (ord == valueCount - 1) {
- nextOffset = totBytes;
- } else {
- nextOffset = ordToOffsetIndex.get(1 + ord);
- }
- final long offset = ordToOffsetIndex.get(ord);
- data.fillSlice(bytesRef, offset, (int) (nextOffset - offset));
- return bytesRef;
- }
+ @Override
+ public BytesRef getByOrd(int ord, BytesRef bytesRef) {
+ final long offset = ordToOffsetIndex.get(ord);
+ final long nextOffset = ordToOffsetIndex.get(1 + ord);
+ data.fillSlice(bytesRef, offset, (int) (nextOffset - offset));
+ return bytesRef;
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new VarSortedBytesEnum(source, cloneData(), cloneIndex());
+ public int getValueCount() {
+ return valueCount;
}
+ }
- private static class VarSortedBytesEnum extends ValuesEnum {
- private PackedInts.Reader docToOrdIndex;
- private PackedInts.Reader ordToOffsetIndex;
- private IndexInput idxIn;
- private IndexInput datIn;
- private int valueCount;
- private long totBytes;
- private int docCount;
- private int pos = -1;
- private final long fp;
-
- protected VarSortedBytesEnum(AttributeSource source, IndexInput datIn,
- IndexInput idxIn) throws IOException {
- super(source, ValueType.BYTES_VAR_SORTED);
- totBytes = idxIn.readLong();
- // keep that in memory to prevent lots of disk seeks
- docToOrdIndex = PackedInts.getReader(idxIn);
- ordToOffsetIndex = PackedInts.getReader(idxIn);
- valueCount = ordToOffsetIndex.size();
- docCount = docToOrdIndex.size();
- fp = datIn.getFilePointer();
- this.idxIn = idxIn;
- this.datIn = datIn;
- }
+ private static final class DirectSortedSource extends SortedSource {
+ private final PackedInts.Reader docToOrdIndex;
+ private final PackedInts.RandomAccessReaderIterator ordToOffsetIndex;
+ private final IndexInput datIn;
+ private final long basePointer;
+ private final int valueCount;
+
+ DirectSortedSource(IndexInput datIn, IndexInput idxIn,
+ Comparator<BytesRef> comparator, ValueType type) throws IOException {
+ super(type, comparator);
+ idxIn.readLong();
+ docToOrdIndex = PackedInts.getReader(idxIn); // read the ords in to prevent too many random disk seeks
+ ordToOffsetIndex = PackedInts.getRandomAccessReaderIterator(idxIn);
+ valueCount = ordToOffsetIndex.size()-1; // the last value here is just a dummy value to get the length of the last value
+ basePointer = datIn.getFilePointer();
+ this.datIn = datIn;
+ }
- @Override
- public void close() throws IOException {
- idxIn.close();
- datIn.close();
- }
+ @Override
+ public int ord(int docID) {
+ return (int) docToOrdIndex.get(docID);
+ }
- @Override
- public int advance(int target) throws IOException {
- if (target >= docCount) {
- return pos = NO_MORE_DOCS;
- }
- int ord;
- while ((ord = (int) docToOrdIndex.get(target)) == 0) {
- if (++target >= docCount) {
- return pos = NO_MORE_DOCS;
- }
- }
- final long offset = ordToOffsetIndex.get(--ord);
- final long nextOffset;
- if (ord == valueCount - 1) {
- nextOffset = totBytes;
- } else {
- nextOffset = ordToOffsetIndex.get(1 + ord);
- }
+ @Override
+ public BytesRef getByOrd(int ord, BytesRef bytesRef) {
+ try {
+ final long offset = ordToOffsetIndex.get(ord);
+ final long nextOffset = ordToOffsetIndex.next();
+ datIn.seek(basePointer + offset);
final int length = (int) (nextOffset - offset);
- datIn.seek(fp + offset);
- if (bytesRef.bytes.length < length)
+ if (bytesRef.bytes.length < length) {
bytesRef.grow(length);
+ }
datIn.readBytes(bytesRef.bytes, 0, length);
bytesRef.length = length;
bytesRef.offset = 0;
- return pos = target;
- }
+ return bytesRef;
+ } catch (IOException ex) {
+ throw new IllegalStateException("failed", ex);
- @Override
- public int docID() {
- return pos;
- }
-
- @Override
- public int nextDoc() throws IOException {
- if (pos >= docCount) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
}
}
-
+
@Override
- public ValueType type() {
- return ValueType.BYTES_VAR_SORTED;
+ public int getValueCount() {
+ return valueCount;
}
+
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Mon Oct 10 19:58:24 2011
@@ -20,18 +20,19 @@ package org.apache.lucene.index.values;
import java.io.IOException;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
+import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
-import org.apache.lucene.index.values.Bytes.DerefBytesSourceBase;
+import org.apache.lucene.index.values.DirectSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
import org.apache.lucene.util.packed.PackedInts;
@@ -66,7 +67,7 @@ class VarStraightBytesImpl {
}
// Fills up to but not including this docID
- private void fill(final int docID) {
+ private void fill(final int docID, final long nextAddress) {
if (docID >= docToAddress.length) {
int oldSize = docToAddress.length;
docToAddress = ArrayUtil.grow(docToAddress, 1 + docID);
@@ -74,7 +75,7 @@ class VarStraightBytesImpl {
* RamUsageEstimator.NUM_BYTES_INT);
}
for (int i = lastDocID + 1; i < docID; i++) {
- docToAddress[i] = address;
+ docToAddress[i] = nextAddress;
}
}
@@ -84,7 +85,7 @@ class VarStraightBytesImpl {
if (bytes.length == 0) {
return; // default
}
- fill(docID);
+ fill(docID, address);
docToAddress[docID] = address;
pool.copy(bytes);
address += bytes.length;
@@ -92,20 +93,20 @@ class VarStraightBytesImpl {
}
@Override
- protected void merge(MergeState state) throws IOException {
+ protected void merge(SingleSubMergeState state) throws IOException {
merge = true;
datOut = getOrCreateDataOut();
boolean success = false;
try {
- if (state.liveDocs == null && state.reader instanceof Reader) {
+ if (state.liveDocs == null && state.reader instanceof VarStraightReader) {
// bulk merge since we don't have any deletes
- Reader reader = (Reader) state.reader;
+ VarStraightReader reader = (VarStraightReader) state.reader;
final int maxDocs = reader.maxDoc;
if (maxDocs == 0) {
return;
}
if (lastDocID+1 < state.docBase) {
- fill(state.docBase);
+ fill(state.docBase, address);
lastDocID = state.docBase-1;
}
final long numDataBytes;
@@ -147,13 +148,14 @@ class VarStraightBytesImpl {
}
@Override
- protected void mergeDoc(int docID) throws IOException {
+ protected void mergeDoc(int docID, int sourceDoc) throws IOException {
assert merge;
assert lastDocID < docID;
+ currentMergeSource.getBytes(sourceDoc, bytesRef);
if (bytesRef.length == 0) {
return; // default
}
- fill(docID);
+ fill(docID, address);
datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
docToAddress[docID] = address;
address += bytesRef.length;
@@ -186,20 +188,21 @@ class VarStraightBytesImpl {
try {
if (lastDocID == -1) {
idxOut.writeVLong(0);
- final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
+ final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount+1,
PackedInts.bitsRequired(0));
- for (int i = 0; i < docCount; i++) {
+ for (int i = 0; i < docCount+1; i++) {
w.add(0);
}
w.finish();
} else {
- fill(docCount);
+ fill(docCount, address);
idxOut.writeVLong(address);
- final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount,
+ final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount+1,
PackedInts.bitsRequired(address));
for (int i = 0; i < docCount; i++) {
w.add(docToAddress[i]);
}
+ w.add(address);
w.finish();
}
success = true;
@@ -220,115 +223,59 @@ class VarStraightBytesImpl {
}
}
- public static class Reader extends BytesReaderBase {
+ public static class VarStraightReader extends BytesReaderBase {
private final int maxDoc;
- Reader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_START, true, context);
+ VarStraightReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_START, true, context, ValueType.BYTES_VAR_STRAIGHT);
this.maxDoc = maxDoc;
}
@Override
public Source load() throws IOException {
- return new Source(cloneData(), cloneIndex());
+ return new VarStraightSource(cloneData(), cloneIndex());
}
- private class Source extends DerefBytesSourceBase {
-
- public Source(IndexInput datIn, IndexInput idxIn) throws IOException {
- super(datIn, idxIn, idxIn.readVLong(), ValueType.BYTES_VAR_STRAIGHT);
- }
-
- @Override
- public BytesRef getBytes(int docID, BytesRef bytesRef) {
- final long address = addresses.get(docID);
- final int length = docID == maxDoc - 1 ? (int) (totalLengthInBytes - address)
- : (int) (addresses.get(1 + docID) - address);
- return data.fillSlice(bytesRef, address, length);
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
- return new SourceEnum(attrSource, type(), this, maxDoc()) {
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs) {
- return pos = NO_MORE_DOCS;
- }
- source.getBytes(target, bytesRef);
- return pos = target;
- }
- };
- }
+ @Override
+ public Source getDirectSource()
+ throws IOException {
+ return new DirectVarStraightSource(cloneData(), cloneIndex(), type());
+ }
+ }
+
+ private static final class VarStraightSource extends BytesSourceBase {
+ private final PackedInts.Reader addresses;
+
+ public VarStraightSource(IndexInput datIn, IndexInput idxIn) throws IOException {
+ super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), idxIn.readVLong(),
+ ValueType.BYTES_VAR_STRAIGHT);
+ addresses = PackedInts.getReader(idxIn);
}
@Override
- public ValuesEnum getEnum(AttributeSource source) throws IOException {
- return new VarStraightBytesEnum(source, cloneData(), cloneIndex());
+ public BytesRef getBytes(int docID, BytesRef bytesRef) {
+ final long address = addresses.get(docID);
+ return data.fillSlice(bytesRef, address,
+ (int) (addresses.get(docID + 1) - address));
}
+ }
+
+ public final static class DirectVarStraightSource extends DirectSource {
- private class VarStraightBytesEnum extends ValuesEnum {
- private final PackedInts.ReaderIterator addresses;
- private final IndexInput datIn;
- private final IndexInput idxIn;
- private final long fp;
- private final long totBytes;
- private int pos = -1;
- private long nextAddress;
-
- protected VarStraightBytesEnum(AttributeSource source, IndexInput datIn,
- IndexInput idxIn) throws IOException {
- super(source, ValueType.BYTES_VAR_STRAIGHT);
- totBytes = idxIn.readVLong();
- fp = datIn.getFilePointer();
- addresses = PackedInts.getReaderIterator(idxIn);
- this.datIn = datIn;
- this.idxIn = idxIn;
- nextAddress = addresses.next();
- }
-
- @Override
- public void close() throws IOException {
- datIn.close();
- idxIn.close();
- }
-
- @Override
- public int advance(final int target) throws IOException {
- if (target >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- final long addr = pos+1 == target ? nextAddress : addresses.advance(target);
- if (addr == totBytes) { // empty values at the end
- bytesRef.length = 0;
- bytesRef.offset = 0;
- return pos = target;
- }
- datIn.seek(fp + addr);
- final int size = (int) (target == maxDoc - 1 ? totBytes - addr
- : (nextAddress = addresses.next()) - addr);
- if (bytesRef.bytes.length < size) {
- bytesRef.grow(size);
- }
- bytesRef.length = size;
- datIn.readBytes(bytesRef.bytes, 0, size);
- return pos = target;
- }
-
- @Override
- public int docID() {
- return pos;
- }
+ private final PackedInts.RandomAccessReaderIterator index;
- @Override
- public int nextDoc() throws IOException {
- return advance(pos + 1);
- }
+ DirectVarStraightSource(IndexInput data, IndexInput index, ValueType type)
+ throws IOException {
+ super(data, type);
+ index.readVLong();
+ this.index = PackedInts.getRandomAccessReaderIterator(index);
}
@Override
- public ValueType type() {
- return ValueType.BYTES_VAR_STRAIGHT;
+ protected int position(int docID) throws IOException {
+ final long offset = index.get(docID);
+ data.seek(baseOffset + offset);
+ return (int) (index.next() - offset);
}
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Writer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Writer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Writer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/Writer.java Mon Oct 10 19:58:24 2011
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.Comparator;
import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
@@ -39,7 +40,7 @@ import org.apache.lucene.util.Counter;
* @lucene.experimental
*/
public abstract class Writer extends DocValuesConsumer {
-
+ protected Source currentMergeSource;
/**
* Creates a new {@link Writer}.
*
@@ -99,31 +100,32 @@ public abstract class Writer extends Doc
}
/**
- * Records a value from the given document id. The methods implementation
- * obtains the value for the document id from the last {@link ValuesEnum}
- * set to {@link #setNextEnum(ValuesEnum)}.
+ * Merges a document with the given <code>docID</code>. The methods
+ * implementation obtains the value for the <i>sourceDoc</i> id from the
+ * current {@link Source} set to <i>setNextMergeSource(Source)</i>.
* <p>
* This method is used during merging to provide implementation agnostic
* default merge implementation.
* </p>
* <p>
- * The given document id must be the same document id returned from
- * {@link ValuesEnum#docID()} when this method is called. All documents IDs
- * between the given ID and the previously given ID or <tt>0</tt> if the
- * method is call the first time are filled with default values depending on
- * the {@link Writer} implementation. The given document ID must always be
- * greater than the previous ID or <tt>0</tt> if called the first time.
+ * All documents IDs between the given ID and the previously given ID or
+ * <tt>0</tt> if the method is call the first time are filled with default
+ * values depending on the {@link Writer} implementation. The given document
+ * ID must always be greater than the previous ID or <tt>0</tt> if called the
+ * first time.
*/
- protected abstract void mergeDoc(int docID) throws IOException;
+ protected abstract void mergeDoc(int docID, int sourceDoc) throws IOException;
/**
- * Sets the next {@link ValuesEnum} to consume values from on calls to
- * {@link #mergeDoc(int)}
+ * Sets the next {@link Source} to consume values from on calls to
+ * {@link #mergeDoc(int, int)}
*
- * @param valuesEnum
- * the next {@link ValuesEnum}, this must not be null
+ * @param mergeSource
+ * the next {@link Source}, this must not be null
*/
- protected abstract void setNextEnum(ValuesEnum valuesEnum);
+ protected void setNextMergeSource(Source mergeSource) {
+ currentMergeSource = mergeSource;
+ }
/**
* Finish writing and close any files and resources used by this Writer.
@@ -136,39 +138,25 @@ public abstract class Writer extends Doc
public abstract void finish(int docCount) throws IOException;
@Override
- protected void merge(MergeState state) throws IOException {
+ protected void merge(SingleSubMergeState state) throws IOException {
// This enables bulk copies in subclasses per MergeState, subclasses can
// simply override this and decide if they want to merge
// segments using this generic implementation or if a bulk merge is possible
// / feasible.
- final ValuesEnum valEnum = state.reader.getEnum();
- assert valEnum != null;
- try {
- setNextEnum(valEnum); // set the current enum we are working on - the
- // impl. will get the correct reference for the type
- // it supports
- int docID = state.docBase;
- final Bits liveDocs = state.liveDocs;
- final int docCount = state.docCount;
- int currentDocId;
- if ((currentDocId = valEnum.advance(0)) != ValuesEnum.NO_MORE_DOCS) {
- for (int i = 0; i < docCount; i++) {
- if (liveDocs == null || liveDocs.get(i)) {
- if (currentDocId < i) {
- if ((currentDocId = valEnum.advance(i)) == ValuesEnum.NO_MORE_DOCS) {
- break; // advance can jump over default values
- }
- }
- if (currentDocId == i) { // we are on the doc to merge
- mergeDoc(docID);
- }
- ++docID;
- }
- }
+ final Source source = state.reader.getDirectSource();
+ assert source != null;
+ setNextMergeSource(source); // set the current enum we are working on - the
+ // impl. will get the correct reference for the type
+ // it supports
+ int docID = state.docBase;
+ final Bits liveDocs = state.liveDocs;
+ final int docCount = state.docCount;
+ for (int i = 0; i < docCount; i++) {
+ if (liveDocs == null || liveDocs.get(i)) {
+ mergeDoc(docID++, i);
}
- } finally {
- valEnum.close();
}
+
}
/**
@@ -182,11 +170,6 @@ public abstract class Writer extends Doc
* the file name id used to create files within the writer.
* @param directory
* the {@link Directory} to create the files from.
- * @param comp
- * a {@link BytesRef} comparator used for {@link Bytes} variants. If
- * <code>null</code>
- * {@link BytesRef#getUTF8SortedAsUnicodeComparator()} is used as the
- * default.
* @param bytesUsed
* a byte-usage tracking reference
* @return a new {@link Writer} instance for the given {@link ValueType}
@@ -205,28 +188,27 @@ public abstract class Writer extends Doc
case VAR_INTS:
return Ints.getWriter(directory, id, bytesUsed, type, context);
case FLOAT_32:
- return Floats.getWriter(directory, id, 4, bytesUsed, context);
+ return Floats.getWriter(directory, id, bytesUsed, context, type);
case FLOAT_64:
- return Floats.getWriter(directory, id, 8, bytesUsed, context);
+ return Floats.getWriter(directory, id, bytesUsed, context, type);
case BYTES_FIXED_STRAIGHT:
- return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, true,
+ return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, comp,
bytesUsed, context);
case BYTES_FIXED_DEREF:
- return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, true,
+ return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, comp,
bytesUsed, context);
case BYTES_FIXED_SORTED:
- return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, true,
+ return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, true, comp,
bytesUsed, context);
case BYTES_VAR_STRAIGHT:
- return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, comp, false,
+ return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, comp,
bytesUsed, context);
case BYTES_VAR_DEREF:
- return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, comp, false,
+ return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, comp,
bytesUsed, context);
case BYTES_VAR_SORTED:
- return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, false,
+ return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, false, comp,
bytesUsed, context);
-
default:
throw new IllegalArgumentException("Unknown Values: " + type);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Mon Oct 10 19:58:24 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs;
@@ -238,7 +239,7 @@ public class BooleanQuery extends Query
for (Iterator<Weight> wIter = weights.iterator(); wIter.hasNext();) {
Weight w = wIter.next();
BooleanClause c = cIter.next();
- if (w.scorer(context, ScorerContext.def().scoreDocsInOrder(true).topScorer(true)) == null) {
+ if (w.scorer(context, true, true, context.reader.getLiveDocs()) == null) {
if (c.isRequired()) {
fail = true;
Explanation r = new Explanation(0.0f, "no match on required clause (" + c.getQuery().toString() + ")");
@@ -300,11 +301,12 @@ public class BooleanQuery extends Query
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext)
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs)
throws IOException {
if (termConjunction) {
// specialized scorer for term conjunctions
- return createConjunctionTermScorer(context);
+ return createConjunctionTermScorer(context, acceptDocs);
}
List<Scorer> required = new ArrayList<Scorer>();
List<Scorer> prohibited = new ArrayList<Scorer>();
@@ -312,7 +314,7 @@ public class BooleanQuery extends Query
Iterator<BooleanClause> cIter = clauses.iterator();
for (Weight w : weights) {
BooleanClause c = cIter.next();
- Scorer subScorer = w.scorer(context, ScorerContext.def());
+ Scorer subScorer = w.scorer(context, true, false, acceptDocs);
if (subScorer == null) {
if (c.isRequired()) {
return null;
@@ -327,7 +329,7 @@ public class BooleanQuery extends Query
}
// Check if we can return a BooleanScorer
- if (!scorerContext.scoreDocsInOrder && scorerContext.topScorer && required.size() == 0 && prohibited.size() < 32) {
+ if (!scoreDocsInOrder && topScorer && required.size() == 0 && prohibited.size() < 32) {
return new BooleanScorer(this, disableCoord, minNrShouldMatch, optional, prohibited, maxCoord);
}
@@ -345,7 +347,7 @@ public class BooleanQuery extends Query
return new BooleanScorer2(this, disableCoord, minNrShouldMatch, required, prohibited, optional, maxCoord);
}
- private Scorer createConjunctionTermScorer(AtomicReaderContext context)
+ private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
throws IOException {
final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
for (int i = 0; i < docsAndFreqs.length; i++) {
@@ -356,7 +358,7 @@ public class BooleanQuery extends Query
}
final ExactDocScorer docScorer = weight.createDocScorer(context);
docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docs(
- context.reader.getLiveDocs(), null), termsEnum.docFreq(), docScorer);
+ acceptDocs, null), termsEnum.docFreq(), docScorer);
}
return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Mon Oct 10 19:58:24 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
@@ -126,7 +127,8 @@ public class ConstantScoreQuery extends
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
final DocIdSetIterator disi;
if (filter != null) {
assert query == null;
@@ -137,7 +139,7 @@ public class ConstantScoreQuery extends
disi = dis.iterator();
} else {
assert query != null && innerWeight != null;
- disi = innerWeight.scorer(context, scorerContext);
+ disi = innerWeight.scorer(context, scoreDocsInOrder, topScorer, acceptDocs);
}
if (disi == null) {
@@ -153,7 +155,7 @@ public class ConstantScoreQuery extends
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- final Scorer cs = scorer(context, ScorerContext.def());
+ final Scorer cs = scorer(context, true, false, context.reader.getLiveDocs());
final boolean exists = (cs != null && cs.advance(doc) == doc);
final ComplexExplanation result = new ComplexExplanation();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java Mon Oct 10 19:58:24 2011
@@ -25,6 +25,7 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Bits;
/**
* A query that generates the union of documents produced by its subqueries, and that scores each document with the maximum
@@ -135,11 +136,13 @@ public class DisjunctionMaxQuery extends
/** Create the scorer used to score our associated DisjunctionMaxQuery */
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
Scorer[] scorers = new Scorer[weights.size()];
int idx = 0;
for (Weight w : weights) {
- Scorer subScorer = w.scorer(context, ScorerContext.def());
+ // we will advance() subscorers
+ Scorer subScorer = w.scorer(context, true, false, acceptDocs);
if (subScorer != null && subScorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
scorers[idx++] = subScorer;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldComparator.java Mon Oct 10 19:58:24 2011
@@ -26,9 +26,6 @@ import org.apache.lucene.search.cache.*;
import org.apache.lucene.search.cache.CachedArray.*;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.packed.Direct16;
-import org.apache.lucene.util.packed.Direct32;
-import org.apache.lucene.util.packed.Direct8;
import org.apache.lucene.util.packed.PackedInts;
import java.io.IOException;
@@ -1222,14 +1219,21 @@ public abstract class FieldComparator<T>
final int docBase = context.docBase;
termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader, field);
final PackedInts.Reader docToOrd = termsIndex.getDocToOrd();
- FieldComparator perSegComp;
- if (docToOrd instanceof Direct8) {
- perSegComp = new ByteOrdComparator(((Direct8) docToOrd).getArray(), termsIndex, docBase);
- } else if (docToOrd instanceof Direct16) {
- perSegComp = new ShortOrdComparator(((Direct16) docToOrd).getArray(), termsIndex, docBase);
- } else if (docToOrd instanceof Direct32) {
- perSegComp = new IntOrdComparator(((Direct32) docToOrd).getArray(), termsIndex, docBase);
- } else {
+ FieldComparator perSegComp = null;
+ if (docToOrd.hasArray()) {
+ final Object arr = docToOrd.getArray();
+ if (arr instanceof byte[]) {
+ perSegComp = new ByteOrdComparator((byte[]) arr, termsIndex, docBase);
+ } else if (arr instanceof short[]) {
+ perSegComp = new ShortOrdComparator((short[]) arr, termsIndex, docBase);
+ } else if (arr instanceof int[]) {
+ perSegComp = new IntOrdComparator((int[]) arr, termsIndex, docBase);
+ }
+ // Don't specialize the long[] case since it's not
+ // possible, ie, worse case is MAX_INT-1 docs with
+ // every one having a unique value.
+ }
+ if (perSegComp == null) {
perSegComp = new AnyOrdComparator(docToOrd, termsIndex, docBase);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FilteredQuery.java Mon Oct 10 19:58:24 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
@@ -66,23 +67,17 @@ extends Query {
@Override
public float getValueForNormalization() throws IOException {
- return weight.getValueForNormalization() * getBoost() * getBoost();
+ return weight.getValueForNormalization() * getBoost() * getBoost(); // boost sub-weight
}
@Override
public void normalize (float norm, float topLevelBoost) {
- weight.normalize(norm, topLevelBoost);
+ weight.normalize(norm, topLevelBoost * getBoost()); // incorporate boost
}
@Override
public Explanation explain (AtomicReaderContext ir, int i) throws IOException {
Explanation inner = weight.explain (ir, i);
- if (getBoost()!=1) {
- Explanation preBoost = inner;
- inner = new Explanation(inner.getValue()*getBoost(),"product of:");
- inner.addDetail(new Explanation(getBoost(),"boost"));
- inner.addDetail(preBoost);
- }
Filter f = FilteredQuery.this.filter;
DocIdSet docIdSet = f.getDocIdSet(ir);
DocIdSetIterator docIdSetIterator = docIdSet == null ? DocIdSet.EMPTY_DOCIDSET.iterator() : docIdSet.iterator();
@@ -105,9 +100,11 @@ extends Query {
// return a filtering scorer
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scoreContext)
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs)
throws IOException {
- final Scorer scorer = weight.scorer(context, ScorerContext.def());
+ // we will advance() the subscorer
+ final Scorer scorer = weight.scorer(context, true, false, acceptDocs);
if (scorer == null) {
return null;
}
@@ -155,7 +152,7 @@ extends Query {
}
@Override
- public float score() throws IOException { return getBoost() * scorer.score(); }
+ public float score() throws IOException { return scorer.score(); }
};
}
};
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Mon Oct 10 19:58:24 2011
@@ -37,7 +37,6 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Weight.ScorerContext;
import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
@@ -569,13 +568,11 @@ public class IndexSearcher implements Cl
// TODO: should we make this
// threaded...? the Collector could be sync'd?
- ScorerContext scorerContext = ScorerContext.def().scoreDocsInOrder(true).topScorer(true);
// always use single thread:
if (filter == null) {
for (int i = 0; i < leaves.length; i++) { // search each subreader
collector.setNextReader(leaves[i]);
- scorerContext = scorerContext.scoreDocsInOrder(!collector.acceptsDocsOutOfOrder());
- Scorer scorer = weight.scorer(leaves[i], scorerContext);
+ Scorer scorer = weight.scorer(leaves[i], !collector.acceptsDocsOutOfOrder(), true, leaves[i].reader.getLiveDocs());
if (scorer != null) {
scorer.score(collector);
}
@@ -593,7 +590,8 @@ public class IndexSearcher implements Cl
assert filter != null;
- Scorer scorer = weight.scorer(context, ScorerContext.def());
+ // we are gonna advance() this scorer, so we set inorder=true/toplevel=false
+ Scorer scorer = weight.scorer(context, true, false, context.reader.getLiveDocs());
if (scorer == null) {
return;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Mon Oct 10 19:58:24 2011
@@ -38,9 +38,9 @@ public class MatchAllDocsQuery extends Q
private final int maxDoc;
private final Bits liveDocs;
- MatchAllScorer(IndexReader reader, Weight w, float score) throws IOException {
+ MatchAllScorer(IndexReader reader, Bits liveDocs, Weight w, float score) throws IOException {
super(w);
- liveDocs = reader.getLiveDocs();
+ this.liveDocs = liveDocs;
this.score = score;
maxDoc = reader.maxDoc();
}
@@ -104,8 +104,9 @@ public class MatchAllDocsQuery extends Q
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new MatchAllScorer(context.reader, this, queryWeight);
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
+ return new MatchAllScorer(context.reader, acceptDocs, this, queryWeight);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Mon Oct 10 19:58:24 2011
@@ -164,10 +164,11 @@ public class MultiPhraseQuery extends Qu
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
assert !termArrays.isEmpty();
final IndexReader reader = context.reader;
- final Bits liveDocs = reader.getLiveDocs();
+ final Bits liveDocs = acceptDocs;
PhraseQuery.PostingsAndFreq[] postingsFreqs = new PhraseQuery.PostingsAndFreq[termArrays.size()];
@@ -227,7 +228,7 @@ public class MultiPhraseQuery extends Qu
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- Scorer scorer = scorer(context, ScorerContext.def());
+ Scorer scorer = scorer(context, true, false, context.reader.getLiveDocs());
if (scorer != null) {
int newDoc = scorer.advance(doc);
if (newDoc == doc) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Mon Oct 10 19:58:24 2011
@@ -212,10 +212,11 @@ public class PhraseQuery extends Query {
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
assert !terms.isEmpty();
final IndexReader reader = context.reader;
- final Bits liveDocs = reader.getLiveDocs();
+ final Bits liveDocs = acceptDocs;
PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()];
for (int i = 0; i < terms.size(); i++) {
final Term t = terms.get(i);
@@ -267,7 +268,7 @@ public class PhraseQuery extends Query {
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- Scorer scorer = scorer(context, ScorerContext.def());
+ Scorer scorer = scorer(context, true, false, context.reader.getLiveDocs());
if (scorer != null) {
int newDoc = scorer.advance(doc);
if (newDoc == doc) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java Mon Oct 10 19:58:24 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.Weight.ScorerContext;
/**
* Constrains search results to only match those which also match a provided
@@ -56,7 +55,7 @@ public class QueryWrapperFilter extends
return new DocIdSet() {
@Override
public DocIdSetIterator iterator() throws IOException {
- return weight.scorer(privateContext, ScorerContext.def());
+ return weight.scorer(privateContext, true, false, privateContext.reader.getLiveDocs());
}
@Override
public boolean isCacheable() { return false; }
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java Mon Oct 10 19:58:24 2011
@@ -61,7 +61,7 @@ public class SpanQueryFilter extends Spa
public SpanFilterResult bitSpans(AtomicReaderContext context) throws IOException {
final FixedBitSet bits = new FixedBitSet(context.reader.maxDoc());
- Spans spans = query.getSpans(context);
+ Spans spans = query.getSpans(context, context.reader.getLiveDocs());
List<SpanFilterResult.PositionInfo> tmp = new ArrayList<SpanFilterResult.PositionInfo>(20);
int currentDoc = -1;
SpanFilterResult.PositionInfo currentInfo = null;