You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/12/10 21:20:13 UTC
svn commit: r1419722 [1/2] - in /lucene/dev/branches/lucene4547/lucene:
codecs/src/java/org/apache/lucene/codecs/memory/
codecs/src/java/org/apache/lucene/codecs/simpletext/
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/index/...
Author: mikemccand
Date: Mon Dec 10 20:20:12 2012
New Revision: 1419722
URL: http://svn.apache.org/viewvc?rev=1419722&view=rev
Log:
cutover TestSort to DV 2.0; remove now unused DV-specific FieldComparators; remove SortField.useIndexValues
Modified:
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValues.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/SortField.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java Mon Dec 10 20:20:12 2012
@@ -56,7 +56,8 @@ public class MemoryDocValuesFormat exten
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
- NumericDocValues valuesIn = producer.getNumeric(field);
+ // nocommit is that clone() ok...? it IS necessary
+ NumericDocValues valuesIn = producer.clone().getNumeric(field);
final int maxDoc = valuesIn.size();
final long minValue = valuesIn.minValue();
@@ -95,7 +96,8 @@ public class MemoryDocValuesFormat exten
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
- BinaryDocValues valuesIn = producer.getBinary(field);
+ // nocommit is that clone() ok...? it IS necessary
+ BinaryDocValues valuesIn = producer.clone().getBinary(field);
final int maxDoc = valuesIn.size();
final int maxLength = valuesIn.maxLength();
final boolean fixedLength = valuesIn.isFixedLength();
@@ -136,7 +138,8 @@ public class MemoryDocValuesFormat exten
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
- SortedDocValues valuesIn = producer.getSorted(field);
+ // nocommit is that clone() ok...? it IS necessary
+ SortedDocValues valuesIn = producer.clone().getSorted(field);
final int maxDoc = valuesIn.size();
final int maxLength = valuesIn.maxLength();
final boolean fixedLength = valuesIn.isFixedLength();
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java Mon Dec 10 20:20:12 2012
@@ -167,6 +167,8 @@ public class SimpleTextSimpleDocValuesFo
@Override
public NumericDocValuesConsumer addNumericField(FieldInfo field, final long minValue, long maxValue) throws IOException {
assert fieldSeen(field.name);
+ assert (field.getDocValuesType() == null || DocValues.isNumber(field.getDocValuesType()) || DocValues.isFloat(field.getDocValuesType())) &&
+ (field.getNormType() == null || DocValues.isNumber(field.getNormType()) || DocValues.isFloat(field.getNormType()));
writeFieldEntry(field);
// write our minimum value to the .dat, all entries are deltas from that
@@ -224,6 +226,7 @@ public class SimpleTextSimpleDocValuesFo
@Override
public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, final int maxLength) throws IOException {
assert fieldSeen(field.name);
+ assert DocValues.isBytes(field.getDocValuesType());
assert !isNorms;
writeFieldEntry(field);
// write fixedlength
@@ -278,6 +281,7 @@ public class SimpleTextSimpleDocValuesFo
@Override
public SortedDocValuesConsumer addSortedField(FieldInfo field, final int valueCount, boolean fixedLength, final int maxLength) throws IOException {
assert fieldSeen(field.name);
+ assert DocValues.isSortedBytes(field.getDocValuesType());
assert !isNorms;
writeFieldEntry(field);
// write numValues
@@ -662,7 +666,7 @@ public class SimpleTextSimpleDocValuesFo
}
in.seek(field.dataStartFilePointer + ord * (9 + field.pattern.length() + field.maxLength));
SimpleTextUtil.readLine(in, scratch);
- assert StringHelper.startsWith(scratch, LENGTH);
+ assert StringHelper.startsWith(scratch, LENGTH): "got " + scratch.utf8ToString() + " in=" + in;
int len;
try {
len = decoder.parse(new String(scratch.bytes, scratch.offset + LENGTH.length, scratch.length - LENGTH.length, "UTF-8")).intValue();
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java Mon Dec 10 20:20:12 2012
@@ -65,6 +65,11 @@ public abstract class FilterCodec extend
}
@Override
+ public SimpleDocValuesFormat simpleDocValuesFormat() {
+ return delegate.simpleDocValuesFormat();
+ }
+
+ @Override
public FieldInfosFormat fieldInfosFormat() {
return delegate.fieldInfosFormat();
}
@@ -80,6 +85,11 @@ public abstract class FilterCodec extend
}
@Override
+ public SimpleNormsFormat simpleNormsFormat() {
+ return delegate.simpleNormsFormat();
+ }
+
+ @Override
public PostingsFormat postingsFormat() {
return delegate.postingsFormat();
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValues.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValues.java Mon Dec 10 20:20:12 2012
@@ -518,8 +518,6 @@ public abstract class DocValues implemen
switch(type) {
case BYTES_VAR_SORTED:
case BYTES_FIXED_SORTED:
- case BYTES_VAR_DEREF:
- case BYTES_FIXED_DEREF:
return true;
default:
return false;
@@ -529,6 +527,8 @@ public abstract class DocValues implemen
// nocommit nuke
public static boolean isBytes(Type type) {
switch(type) {
+ case BYTES_VAR_DEREF:
+ case BYTES_FIXED_DEREF:
case BYTES_VAR_STRAIGHT:
case BYTES_FIXED_STRAIGHT:
return true;
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java Mon Dec 10 20:20:12 2012
@@ -49,32 +49,22 @@ final class DocValuesProcessor extends S
public void addField(int docID, StorableField field, FieldInfo fieldInfo) {
final DocValues.Type dvType = field.fieldType().docValueType();
if (dvType != null) {
- switch(dvType) {
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_STRAIGHT:
+ if (DocValues.isBytes(dvType)) {
addBinaryField(fieldInfo, docID, field.binaryValue());
- break;
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_DEREF:
- case BYTES_FIXED_DEREF:
+ } else if (DocValues.isSortedBytes(dvType)) {
addSortedField(fieldInfo, docID, field.binaryValue());
- break;
- case VAR_INTS:
- case FIXED_INTS_8:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
+ } else if (DocValues.isFloat(dvType)) {
+ if (dvType == DocValues.Type.FLOAT_32) {
+ addNumericField(fieldInfo, docID, field.numericValue().floatValue());
+ } else if (dvType == DocValues.Type.FLOAT_64) {
+ addNumericField(fieldInfo, docID, field.numericValue().doubleValue());
+ } else {
+ assert false;
+ }
+ } else if (DocValues.isNumber(dvType)) {
addNumericField(fieldInfo, docID, field.numericValue().longValue());
- break;
- case FLOAT_32:
- addNumericField(fieldInfo, docID, field.numericValue().floatValue());
- break;
- case FLOAT_64:
- addNumericField(fieldInfo, docID, field.numericValue().doubleValue());
- break;
- default:
- break;
+ } else {
+ assert false: "unrecognized DocValues.Type: " + dvType;
}
}
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Mon Dec 10 20:20:12 2012
@@ -1321,6 +1321,10 @@ class FieldCacheImpl implements FieldCac
throws IOException {
BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
+
+ // TODO: would be nice to fallback to SortedDV if it's
+ // available but BinaryDV isn't?
+
if (valuesIn != null) {
return valuesIn;
} else {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java Mon Dec 10 20:20:12 2012
@@ -18,12 +18,10 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.index.AtomicReader; // javadocs
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache.ByteParser;
import org.apache.lucene.search.FieldCache.DoubleParser;
@@ -33,7 +31,6 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.search.FieldCache.ShortParser;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.packed.PackedInts;
/**
* Expert: a FieldComparator compares hits so as to determine their
@@ -389,83 +386,6 @@ public abstract class FieldComparator<T>
}
}
- /** Uses float index values to sort by ascending value */
- public static final class FloatDocValuesComparator extends FieldComparator<Double> {
- private final double[] values;
- private final String field;
- private DocValues.Source currentReaderValues;
- private double bottom;
-
- FloatDocValuesComparator(int numHits, String field) {
- values = new double[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- final double v1 = values[slot1];
- final double v2 = values[slot2];
- if (v1 > v2) {
- return 1;
- } else if (v1 < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public int compareBottom(int doc) {
- final double v2 = currentReaderValues.getFloat(doc);
- if (bottom > v2) {
- return 1;
- } else if (bottom < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- values[slot] = currentReaderValues.getFloat(doc);
- }
-
- @Override
- public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
- final DocValues docValues = context.reader().docValues(field);
- if (docValues != null) {
- currentReaderValues = docValues.getSource();
- } else {
- currentReaderValues = DocValues.getDefaultSource(DocValues.Type.FLOAT_64);
- }
- return this;
- }
-
- @Override
- public void setBottom(final int bottom) {
- this.bottom = values[bottom];
- }
-
- @Override
- public Double value(int slot) {
- return Double.valueOf(values[slot]);
- }
-
- @Override
- public int compareDocToValue(int doc, Double valueObj) {
- final double value = valueObj.doubleValue();
- final double docValue = currentReaderValues.getFloat(doc);
- if (docValue < value) {
- return -1;
- } else if (docValue > value) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
/** Parses field's values as float (using {@link
* FieldCache#getFloats} and sorts by ascending value */
public static final class FloatComparator extends NumericComparator<Float> {
@@ -739,87 +659,6 @@ public abstract class FieldComparator<T>
}
}
- /** Loads int index values and sorts by ascending value. */
- public static final class IntDocValuesComparator extends FieldComparator<Long> {
- private final long[] values;
- private DocValues.Source currentReaderValues;
- private final String field;
- private long bottom;
-
- IntDocValuesComparator(int numHits, String field) {
- values = new long[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- // TODO: there are sneaky non-branch ways to compute
- // -1/+1/0 sign
- final long v1 = values[slot1];
- final long v2 = values[slot2];
- if (v1 > v2) {
- return 1;
- } else if (v1 < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public int compareBottom(int doc) {
- // TODO: there are sneaky non-branch ways to compute
- // -1/+1/0 sign
- final long v2 = currentReaderValues.getInt(doc);
- if (bottom > v2) {
- return 1;
- } else if (bottom < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- values[slot] = currentReaderValues.getInt(doc);
- }
-
- @Override
- public FieldComparator<Long> setNextReader(AtomicReaderContext context) throws IOException {
- DocValues docValues = context.reader().docValues(field);
- if (docValues != null) {
- currentReaderValues = docValues.getSource();
- } else {
- currentReaderValues = DocValues.getDefaultSource(DocValues.Type.FIXED_INTS_64);
- }
- return this;
- }
-
- @Override
- public void setBottom(final int bottom) {
- this.bottom = values[bottom];
- }
-
- @Override
- public Long value(int slot) {
- return Long.valueOf(values[slot]);
- }
-
- @Override
- public int compareDocToValue(int doc, Long valueObj) {
- final long value = valueObj.longValue();
- final long docValue = currentReaderValues.getInt(doc);
- if (docValue < value) {
- return -1;
- } else if (docValue > value) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
/** Parses field's values as long (using {@link
* FieldCache#getLongs} and sorts by ascending value */
public static final class LongComparator extends NumericComparator<Long> {
@@ -1313,75 +1152,25 @@ public abstract class FieldComparator<T>
}
}
- /** Sorts by field's natural Term sort order, using
- * ordinals; this is just like {@link
- * org.apache.lucene.search.FieldComparator.TermValComparator} except it uses DocValues to
- * retrieve the sort ords saved during indexing. */
- public static final class TermOrdValDocValuesComparator extends FieldComparator<BytesRef> {
- /* Ords for each slot.
- @lucene.internal */
- final int[] ords;
-
- /* Values for each slot.
- @lucene.internal */
- final BytesRef[] values;
-
- /* Which reader last copied a value into the slot. When
- we compare two slots, we just compare-by-ord if the
- readerGen is the same; else we must compare the
- values (slower).
- @lucene.internal */
- final int[] readerGen;
-
- /* Gen of current reader we are on.
- @lucene.internal */
- int currentReaderGen = -1;
-
- /* Current reader's doc ord/values.
- @lucene.internal */
- DocValues.SortedSource termsIndex;
-
- /* Comparator for comparing by value.
- @lucene.internal */
- Comparator<BytesRef> comp;
+ /** Sorts by field's natural Term sort order. All
+ * comparisons are done using BytesRef.compareTo, which is
+ * slow for medium to large result sets but possibly
+ * very fast for very small results sets. */
+ public static final class TermValComparator extends FieldComparator<BytesRef> {
+ private BytesRef[] values;
+ private BinaryDocValues docTerms;
private final String field;
+ private BytesRef bottom;
+ private final BytesRef tempBR = new BytesRef();
- /* Bottom slot, or -1 if queue isn't full yet
- @lucene.internal */
- int bottomSlot = -1;
-
- /* Bottom ord (same as ords[bottomSlot] once bottomSlot
- is set). Cached for faster compares.
- @lucene.internal */
- int bottomOrd;
-
- /* True if current bottom slot matches the current
- reader.
- @lucene.internal */
- boolean bottomSameReader;
-
- /* Bottom value (same as values[bottomSlot] once
- bottomSlot is set). Cached for faster compares.
- @lucene.internal */
- BytesRef bottomValue;
-
- /** @lucene.internal */
- final BytesRef tempBR = new BytesRef();
-
- public TermOrdValDocValuesComparator(int numHits, String field) {
- ords = new int[numHits];
+ TermValComparator(int numHits, String field) {
values = new BytesRef[numHits];
- readerGen = new int[numHits];
this.field = field;
}
@Override
public int compare(int slot1, int slot2) {
- if (readerGen[slot1] == readerGen[slot2]) {
- return ords[slot1] - ords[slot2];
- }
-
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1 == null) {
@@ -1392,445 +1181,60 @@ public abstract class FieldComparator<T>
} else if (val2 == null) {
return 1;
}
- return comp.compare(val1, val2);
+
+ return val1.compareTo(val2);
}
@Override
public int compareBottom(int doc) {
- throw new UnsupportedOperationException();
+ docTerms.get(doc, tempBR);
+ if (bottom.bytes == BinaryDocValues.MISSING) {
+ if (tempBR.bytes == BinaryDocValues.MISSING) {
+ return 0;
+ }
+ return -1;
+ } else if (tempBR.bytes == BinaryDocValues.MISSING) {
+ return 1;
+ }
+ return bottom.compareTo(tempBR);
}
@Override
public void copy(int slot, int doc) {
- throw new UnsupportedOperationException();
+ if (values[slot] == null) {
+ values[slot] = new BytesRef();
+ }
+ docTerms.get(doc, values[slot]);
}
@Override
- public int compareDocToValue(int doc, BytesRef value) {
- return termsIndex.getBytes(doc, tempBR).compareTo(value);
+ public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
+ docTerms = FieldCache.DEFAULT.getTerms(context.reader(), field);
+ return this;
+ }
+
+ @Override
+ public void setBottom(final int bottom) {
+ this.bottom = values[bottom];
}
- // TODO: would be nice to share these specialized impls
- // w/ TermOrdValComparator
-
- /** Base class for specialized (per bit width of the
- * ords) per-segment comparator. NOTE: this is messy;
- * we do this only because hotspot can't reliably inline
- * the underlying array access when looking up doc->ord
- * @lucene.internal
- */
- abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
-
- @Override
- public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- return TermOrdValDocValuesComparator.this.setNextReader(context);
- }
+ @Override
+ public BytesRef value(int slot) {
+ return values[slot];
+ }
- @Override
- public int compare(int slot1, int slot2) {
- return TermOrdValDocValuesComparator.this.compare(slot1, slot2);
+ @Override
+ public int compareValues(BytesRef val1, BytesRef val2) {
+ if (val1 == null) {
+ if (val2 == null) {
+ return 0;
+ }
+ return -1;
+ } else if (val2 == null) {
+ return 1;
}
-
- @Override
- public void setBottom(final int bottom) {
- TermOrdValDocValuesComparator.this.setBottom(bottom);
- }
-
- @Override
- public BytesRef value(int slot) {
- return TermOrdValDocValuesComparator.this.value(slot);
- }
-
- @Override
- public int compareValues(BytesRef val1, BytesRef val2) {
- assert val1 != null;
- assert val2 != null;
- return comp.compare(val1, val2);
- }
-
- @Override
- public int compareDocToValue(int doc, BytesRef value) {
- return TermOrdValDocValuesComparator.this.compareDocToValue(doc, value);
- }
- }
-
- // Used per-segment when bit width of doc->ord is 8:
- private final class ByteOrdComparator extends PerSegmentComparator {
- private final byte[] readerOrds;
- private final DocValues.SortedSource termsIndex;
- private final int docBase;
-
- public ByteOrdComparator(byte[] readerOrds, DocValues.SortedSource termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = readerOrds[doc]&0xFF;
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc]&0xFF;
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width of doc->ord is 16:
- private final class ShortOrdComparator extends PerSegmentComparator {
- private final short[] readerOrds;
- private final DocValues.SortedSource termsIndex;
- private final int docBase;
-
- public ShortOrdComparator(short[] readerOrds, DocValues.SortedSource termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = readerOrds[doc]&0xFFFF;
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc]&0xFFFF;
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width of doc->ord is 32:
- private final class IntOrdComparator extends PerSegmentComparator {
- private final int[] readerOrds;
- private final DocValues.SortedSource termsIndex;
- private final int docBase;
-
- public IntOrdComparator(int[] readerOrds, DocValues.SortedSource termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = readerOrds[doc];
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc];
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width is not a native array
- // size (8, 16, 32):
- private final class AnyPackedDocToOrdComparator extends PerSegmentComparator {
- private final PackedInts.Reader readerOrds;
- private final int docBase;
-
- public AnyPackedDocToOrdComparator(PackedInts.Reader readerOrds, int docBase) {
- this.readerOrds = readerOrds;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = (int) readerOrds.get(doc);
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = (int) readerOrds.get(doc);
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when DV doesn't use packed ints for
- // docToOrds:
- private final class AnyOrdComparator extends PerSegmentComparator {
- private final int docBase;
-
- public AnyOrdComparator(int docBase) {
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- final int docOrd = termsIndex.ord(doc);
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = termsIndex.ord(doc);
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- @Override
- public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- final int docBase = context.docBase;
-
- final DocValues dv = context.reader().docValues(field);
- if (dv == null) {
- // This may mean entire segment had no docs with
- // this DV field; use default field value (empty
- // byte[]) in this case:
- termsIndex = DocValues.getDefaultSortedSource(DocValues.Type.BYTES_VAR_SORTED, context.reader().maxDoc());
- } else {
- termsIndex = dv.getSource().asSortedSource();
- if (termsIndex == null) {
- // This means segment has doc values, but they are
- // not able to provide a sorted source; consider
- // this a hard error:
- throw new IllegalStateException("DocValues exist for field \"" + field + "\", but not as a sorted source: type=" + dv.getSource().getType() + " reader=" + context.reader());
- }
- }
-
- comp = termsIndex.getComparator();
-
- FieldComparator<BytesRef> perSegComp = null;
- if (termsIndex.hasPackedDocToOrd()) {
- final PackedInts.Reader docToOrd = termsIndex.getDocToOrd();
- if (docToOrd.hasArray()) {
- final Object arr = docToOrd.getArray();
- assert arr != null;
- if (arr instanceof byte[]) {
- // 8 bit packed
- perSegComp = new ByteOrdComparator((byte[]) arr, termsIndex, docBase);
- } else if (arr instanceof short[]) {
- // 16 bit packed
- perSegComp = new ShortOrdComparator((short[]) arr, termsIndex, docBase);
- } else if (arr instanceof int[]) {
- // 32 bit packed
- perSegComp = new IntOrdComparator((int[]) arr, termsIndex, docBase);
- }
- }
-
- if (perSegComp == null) {
- perSegComp = new AnyPackedDocToOrdComparator(docToOrd, docBase);
- }
- } else {
- if (perSegComp == null) {
- perSegComp = new AnyOrdComparator(docBase);
- }
- }
-
- currentReaderGen++;
- if (bottomSlot != -1) {
- perSegComp.setBottom(bottomSlot);
- }
-
- return perSegComp;
- }
-
- @Override
- public void setBottom(final int bottom) {
- bottomSlot = bottom;
-
- bottomValue = values[bottomSlot];
- if (currentReaderGen == readerGen[bottomSlot]) {
- bottomOrd = ords[bottomSlot];
- bottomSameReader = true;
- } else {
- if (bottomValue == null) {
- // 0 ord is null for all segments
- assert ords[bottomSlot] == 0;
- bottomOrd = 0;
- bottomSameReader = true;
- readerGen[bottomSlot] = currentReaderGen;
- } else {
- final int index = termsIndex.getOrdByValue(bottomValue, tempBR);
- if (index < 0) {
- bottomOrd = -index - 2;
- bottomSameReader = false;
- } else {
- bottomOrd = index;
- // exact value match
- bottomSameReader = true;
- readerGen[bottomSlot] = currentReaderGen;
- ords[bottomSlot] = bottomOrd;
- }
- }
- }
- }
-
- @Override
- public BytesRef value(int slot) {
- return values[slot];
- }
- }
-
- /** Sorts by field's natural Term sort order. All
- * comparisons are done using BytesRef.compareTo, which is
- * slow for medium to large result sets but possibly
- * very fast for very small results sets. */
- public static final class TermValComparator extends FieldComparator<BytesRef> {
-
- private BytesRef[] values;
- private BinaryDocValues docTerms;
- private final String field;
- private BytesRef bottom;
- private final BytesRef tempBR = new BytesRef();
-
- TermValComparator(int numHits, String field) {
- values = new BytesRef[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- final BytesRef val1 = values[slot1];
- final BytesRef val2 = values[slot2];
- if (val1 == null) {
- if (val2 == null) {
- return 0;
- }
- return -1;
- } else if (val2 == null) {
- return 1;
- }
-
- return val1.compareTo(val2);
- }
-
- @Override
- public int compareBottom(int doc) {
- docTerms.get(doc, tempBR);
- if (bottom.bytes == BinaryDocValues.MISSING) {
- if (tempBR.bytes == BinaryDocValues.MISSING) {
- return 0;
- }
- return -1;
- } else if (tempBR.bytes == BinaryDocValues.MISSING) {
- return 1;
- }
- return bottom.compareTo(tempBR);
- }
-
- @Override
- public void copy(int slot, int doc) {
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- docTerms.get(doc, values[slot]);
- }
-
- @Override
- public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- docTerms = FieldCache.DEFAULT.getTerms(context.reader(), field);
- return this;
- }
-
- @Override
- public void setBottom(final int bottom) {
- this.bottom = values[bottom];
- }
-
- @Override
- public BytesRef value(int slot) {
- return values[slot];
- }
-
- @Override
- public int compareValues(BytesRef val1, BytesRef val2) {
- if (val1 == null) {
- if (val2 == null) {
- return 0;
- }
- return -1;
- } else if (val2 == null) {
- return 1;
- }
- return val1.compareTo(val2);
- }
+ return val1.compareTo(val2);
+ }
@Override
public int compareDocToValue(int doc, BytesRef value) {
@@ -1838,77 +1242,4 @@ public abstract class FieldComparator<T>
return tempBR.compareTo(value);
}
}
-
- /** Sorts by field's natural Term sort order. All
- * comparisons are done using BytesRef.compareTo, which is
- * slow for medium to large result sets but possibly
- * very fast for very small results sets. The BytesRef
- * values are obtained using {@link AtomicReader#docValues}. */
- public static final class TermValDocValuesComparator extends FieldComparator<BytesRef> {
-
- private BytesRef[] values;
- private DocValues.Source docTerms;
- private final String field;
- private BytesRef bottom;
- private final BytesRef tempBR = new BytesRef();
-
- TermValDocValuesComparator(int numHits, String field) {
- values = new BytesRef[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- assert values[slot1] != null;
- assert values[slot2] != null;
- return values[slot1].compareTo(values[slot2]);
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottom != null;
- return bottom.compareTo(docTerms.getBytes(doc, tempBR));
- }
-
- @Override
- public void copy(int slot, int doc) {
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- docTerms.getBytes(doc, values[slot]);
- }
-
- @Override
- public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- final DocValues dv = context.reader().docValues(field);
- if (dv != null) {
- docTerms = dv.getSource();
- } else {
- docTerms = DocValues.getDefaultSource(DocValues.Type.BYTES_VAR_DEREF);
- }
- return this;
- }
-
- @Override
- public void setBottom(final int bottom) {
- this.bottom = values[bottom];
- }
-
- @Override
- public BytesRef value(int slot) {
- return values[slot];
- }
-
- @Override
- public int compareValues(BytesRef val1, BytesRef val2) {
- assert val1 != null;
- assert val2 != null;
- return val1.compareTo(val2);
- }
-
- @Override
- public int compareDocToValue(int doc, BytesRef value) {
- return docTerms.getBytes(doc, tempBR).compareTo(value);
- }
- }
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/SortField.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/SortField.java Mon Dec 10 20:20:12 2012
@@ -257,7 +257,6 @@ public class SortField {
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
- String dv = useIndexValues ? " [dv]" : "";
switch (type) {
case SCORE:
buffer.append("<score>");
@@ -268,11 +267,11 @@ public class SortField {
break;
case STRING:
- buffer.append("<string" + dv + ": \"").append(field).append("\">");
+ buffer.append("<string" + ": \"").append(field).append("\">");
break;
case STRING_VAL:
- buffer.append("<string_val" + dv + ": \"").append(field).append("\">");
+ buffer.append("<string_val" + ": \"").append(field).append("\">");
break;
case BYTE:
@@ -284,7 +283,7 @@ public class SortField {
break;
case INT:
- buffer.append("<int" + dv + ": \"").append(field).append("\">");
+ buffer.append("<int" + ": \"").append(field).append("\">");
break;
case LONG:
@@ -292,11 +291,11 @@ public class SortField {
break;
case FLOAT:
- buffer.append("<float" + dv + ": \"").append(field).append("\">");
+ buffer.append("<float" + ": \"").append(field).append("\">");
break;
case DOUBLE:
- buffer.append("<double" + dv + ": \"").append(field).append("\">");
+ buffer.append("<double" + ": \"").append(field).append("\">");
break;
case CUSTOM:
@@ -347,16 +346,6 @@ public class SortField {
return hash;
}
- private boolean useIndexValues;
-
- public void setUseIndexValues(boolean b) {
- useIndexValues = b;
- }
-
- public boolean getUseIndexValues() {
- return useIndexValues;
- }
-
private Comparator<BytesRef> bytesComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
public void setBytesComparator(Comparator<BytesRef> b) {
@@ -389,18 +378,10 @@ public class SortField {
return new FieldComparator.DocComparator(numHits);
case INT:
- if (useIndexValues) {
- return new FieldComparator.IntDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.IntComparator(numHits, field, parser, (Integer) missingValue);
- }
+ return new FieldComparator.IntComparator(numHits, field, parser, (Integer) missingValue);
case FLOAT:
- if (useIndexValues) {
- return new FieldComparator.FloatDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.FloatComparator(numHits, field, parser, (Float) missingValue);
- }
+ return new FieldComparator.FloatComparator(numHits, field, parser, (Float) missingValue);
case LONG:
return new FieldComparator.LongComparator(numHits, field, parser, (Long) missingValue);
@@ -419,18 +400,10 @@ public class SortField {
return comparatorSource.newComparator(field, numHits, sortPos, reverse);
case STRING:
- if (useIndexValues) {
- return new FieldComparator.TermOrdValDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.TermOrdValComparator(numHits, field);
- }
+ return new FieldComparator.TermOrdValComparator(numHits, field);
case STRING_VAL:
- if (useIndexValues) {
- return new FieldComparator.TermValDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.TermValComparator(numHits, field);
- }
+ return new FieldComparator.TermValComparator(numHits, field);
case REWRITEABLE:
throw new IllegalStateException("SortField needs to be rewritten through Sort.rewrite(..) and SortField.rewrite(..)");
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java?rev=1419722&r1=1419721&r2=1419722&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java Mon Dec 10 20:20:12 2012
@@ -47,11 +47,6 @@ public class TestSearchAfter extends Luc
private IndexReader reader;
private IndexSearcher searcher;
- private static SortField useDocValues(SortField field) {
- field.setUseIndexValues(true);
- return field;
- }
-
@Override
public void setUp() throws Exception {
super.setUp();
@@ -125,11 +120,11 @@ public class TestSearchAfter extends Luc
assertQuery(query, filter, new Sort(new SortField[] {new SortField("double", SortField.Type.DOUBLE, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytes", SortField.Type.STRING, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytesval", SortField.Type.STRING_VAL, reversed)}));
- assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("intdocvalues", SortField.Type.INT, reversed))}));
- assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("floatdocvalues", SortField.Type.FLOAT, reversed))}));
- assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("sortedbytesdocvalues", SortField.Type.STRING, reversed))}));
- assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, reversed))}));
- assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, reversed))}));
+ assertQuery(query, filter, new Sort(new SortField[] {new SortField("intdocvalues", SortField.Type.INT, reversed)}));
+ assertQuery(query, filter, new Sort(new SortField[] {new SortField("floatdocvalues", SortField.Type.FLOAT, reversed)}));
+ assertQuery(query, filter, new Sort(new SortField[] {new SortField("sortedbytesdocvalues", SortField.Type.STRING, reversed)}));
+ assertQuery(query, filter, new Sort(new SortField[] {new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, reversed)}));
+ assertQuery(query, filter, new Sort(new SortField[] {new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, reversed)}));
}
}