You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/07 21:48:28 UTC
svn commit: r1443717 [5/14] - in /lucene/dev/trunk: ./ dev-tools/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/icu/src/java/org/apache/lucene/collation/
lucene/analysis/icu/src/test/org/apache/lucene/collation/ lucene/backwards/
luc...
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java Thu Feb 7 20:48:21 2013
@@ -18,14 +18,12 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Comparator;
import org.apache.lucene.index.AtomicReader; // javadocs
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache.ByteParser;
-import org.apache.lucene.search.FieldCache.DocTerms;
-import org.apache.lucene.search.FieldCache.DocTermsIndex;
import org.apache.lucene.search.FieldCache.DoubleParser;
import org.apache.lucene.search.FieldCache.FloatParser;
import org.apache.lucene.search.FieldCache.IntParser;
@@ -33,7 +31,6 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.search.FieldCache.ShortParser;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.packed.PackedInts;
/**
* Expert: a FieldComparator compares hits so as to determine their
@@ -227,7 +224,7 @@ public abstract class FieldComparator<T>
public static final class ByteComparator extends NumericComparator<Byte> {
private final byte[] values;
private final ByteParser parser;
- private byte[] currentReaderValues;
+ private FieldCache.Bytes currentReaderValues;
private byte bottom;
ByteComparator(int numHits, String field, FieldCache.Parser parser, Byte missingValue) {
@@ -243,7 +240,7 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
- byte v2 = currentReaderValues[doc];
+ byte v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -255,7 +252,7 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
- byte v2 = currentReaderValues[doc];
+ byte v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -284,7 +281,7 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, Byte value) {
- byte docValue = currentReaderValues[doc];
+ byte docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
@@ -299,7 +296,7 @@ public abstract class FieldComparator<T>
public static final class DoubleComparator extends NumericComparator<Double> {
private final double[] values;
private final DoubleParser parser;
- private double[] currentReaderValues;
+ private FieldCache.Doubles currentReaderValues;
private double bottom;
DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue) {
@@ -323,7 +320,7 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
- double v2 = currentReaderValues[doc];
+ double v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -341,7 +338,7 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
- double v2 = currentReaderValues[doc];
+ double v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -372,7 +369,7 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, Double valueObj) {
final double value = valueObj.doubleValue();
- double docValue = currentReaderValues[doc];
+ double docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
@@ -388,89 +385,12 @@ public abstract class FieldComparator<T>
}
}
- /** Uses float index values to sort by ascending value */
- public static final class FloatDocValuesComparator extends FieldComparator<Double> {
- private final double[] values;
- private final String field;
- private DocValues.Source currentReaderValues;
- private double bottom;
-
- FloatDocValuesComparator(int numHits, String field) {
- values = new double[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- final double v1 = values[slot1];
- final double v2 = values[slot2];
- if (v1 > v2) {
- return 1;
- } else if (v1 < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public int compareBottom(int doc) {
- final double v2 = currentReaderValues.getFloat(doc);
- if (bottom > v2) {
- return 1;
- } else if (bottom < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- values[slot] = currentReaderValues.getFloat(doc);
- }
-
- @Override
- public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
- final DocValues docValues = context.reader().docValues(field);
- if (docValues != null) {
- currentReaderValues = docValues.getSource();
- } else {
- currentReaderValues = DocValues.getDefaultSource(DocValues.Type.FLOAT_64);
- }
- return this;
- }
-
- @Override
- public void setBottom(final int bottom) {
- this.bottom = values[bottom];
- }
-
- @Override
- public Double value(int slot) {
- return Double.valueOf(values[slot]);
- }
-
- @Override
- public int compareDocToValue(int doc, Double valueObj) {
- final double value = valueObj.doubleValue();
- final double docValue = currentReaderValues.getFloat(doc);
- if (docValue < value) {
- return -1;
- } else if (docValue > value) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
/** Parses field's values as float (using {@link
* FieldCache#getFloats} and sorts by ascending value */
public static final class FloatComparator extends NumericComparator<Float> {
private final float[] values;
private final FloatParser parser;
- private float[] currentReaderValues;
+ private FieldCache.Floats currentReaderValues;
private float bottom;
FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue) {
@@ -497,7 +417,7 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
// TODO: are there sneaky non-branch ways to compute sign of float?
- float v2 = currentReaderValues[doc];
+ float v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -515,7 +435,7 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
- float v2 = currentReaderValues[doc];
+ float v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -546,7 +466,7 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, Float valueObj) {
final float value = valueObj.floatValue();
- float docValue = currentReaderValues[doc];
+ float docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
@@ -567,7 +487,7 @@ public abstract class FieldComparator<T>
public static final class ShortComparator extends NumericComparator<Short> {
private final short[] values;
private final ShortParser parser;
- private short[] currentReaderValues;
+ private FieldCache.Shorts currentReaderValues;
private short bottom;
ShortComparator(int numHits, String field, FieldCache.Parser parser, Short missingValue) {
@@ -583,7 +503,7 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
- short v2 = currentReaderValues[doc];
+ short v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -595,7 +515,7 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
- short v2 = currentReaderValues[doc];
+ short v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -626,7 +546,7 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, Short valueObj) {
final short value = valueObj.shortValue();
- short docValue = currentReaderValues[doc];
+ short docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
@@ -641,7 +561,7 @@ public abstract class FieldComparator<T>
public static final class IntComparator extends NumericComparator<Integer> {
private final int[] values;
private final IntParser parser;
- private int[] currentReaderValues;
+ private FieldCache.Ints currentReaderValues;
private int bottom; // Value of bottom of queue
IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue) {
@@ -673,7 +593,7 @@ public abstract class FieldComparator<T>
// -1/+1/0 sign
// Cannot return bottom - values[slot2] because that
// may overflow
- int v2 = currentReaderValues[doc];
+ int v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -691,7 +611,7 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
- int v2 = currentReaderValues[doc];
+ int v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -722,7 +642,7 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
- int docValue = currentReaderValues[doc];
+ int docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
@@ -738,93 +658,12 @@ public abstract class FieldComparator<T>
}
}
- /** Loads int index values and sorts by ascending value. */
- public static final class IntDocValuesComparator extends FieldComparator<Long> {
- private final long[] values;
- private DocValues.Source currentReaderValues;
- private final String field;
- private long bottom;
-
- IntDocValuesComparator(int numHits, String field) {
- values = new long[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- // TODO: there are sneaky non-branch ways to compute
- // -1/+1/0 sign
- final long v1 = values[slot1];
- final long v2 = values[slot2];
- if (v1 > v2) {
- return 1;
- } else if (v1 < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public int compareBottom(int doc) {
- // TODO: there are sneaky non-branch ways to compute
- // -1/+1/0 sign
- final long v2 = currentReaderValues.getInt(doc);
- if (bottom > v2) {
- return 1;
- } else if (bottom < v2) {
- return -1;
- } else {
- return 0;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- values[slot] = currentReaderValues.getInt(doc);
- }
-
- @Override
- public FieldComparator<Long> setNextReader(AtomicReaderContext context) throws IOException {
- DocValues docValues = context.reader().docValues(field);
- if (docValues != null) {
- currentReaderValues = docValues.getSource();
- } else {
- currentReaderValues = DocValues.getDefaultSource(DocValues.Type.FIXED_INTS_64);
- }
- return this;
- }
-
- @Override
- public void setBottom(final int bottom) {
- this.bottom = values[bottom];
- }
-
- @Override
- public Long value(int slot) {
- return Long.valueOf(values[slot]);
- }
-
- @Override
- public int compareDocToValue(int doc, Long valueObj) {
- final long value = valueObj.longValue();
- final long docValue = currentReaderValues.getInt(doc);
- if (docValue < value) {
- return -1;
- } else if (docValue > value) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
/** Parses field's values as long (using {@link
* FieldCache#getLongs} and sorts by ascending value */
public static final class LongComparator extends NumericComparator<Long> {
private final long[] values;
private final LongParser parser;
- private long[] currentReaderValues;
+ private FieldCache.Longs currentReaderValues;
private long bottom;
LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue) {
@@ -852,7 +691,7 @@ public abstract class FieldComparator<T>
public int compareBottom(int doc) {
// TODO: there are sneaky non-branch ways to compute
// -1/+1/0 sign
- long v2 = currentReaderValues[doc];
+ long v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -870,7 +709,7 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
- long v2 = currentReaderValues[doc];
+ long v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
@@ -901,7 +740,7 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, Long valueObj) {
final long value = valueObj.longValue();
- long docValue = currentReaderValues[doc];
+ long docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
@@ -1095,7 +934,7 @@ public abstract class FieldComparator<T>
/* Current reader's doc ord/values.
@lucene.internal */
- DocTermsIndex termsIndex;
+ SortedDocValues termsIndex;
private final String field;
@@ -1158,8 +997,8 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, BytesRef value) {
- BytesRef docValue = termsIndex.getTerm(doc, tempBR);
- if (docValue == null) {
+ int ord = termsIndex.getOrd(doc);
+ if (ord == -1) {
if (value == null) {
return 0;
}
@@ -1167,7 +1006,8 @@ public abstract class FieldComparator<T>
} else if (value == null) {
return 1;
}
- return docValue.compareTo(value);
+ termsIndex.lookupOrd(ord, tempBR);
+ return tempBR.compareTo(value);
}
/** Base class for specialized (per bit width of the
@@ -1217,153 +1057,12 @@ public abstract class FieldComparator<T>
}
}
- // Used per-segment when bit width of doc->ord is 8:
- private final class ByteOrdComparator extends PerSegmentComparator {
- private final byte[] readerOrds;
- private final DocTermsIndex termsIndex;
- private final int docBase;
-
- public ByteOrdComparator(byte[] readerOrds, DocTermsIndex termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = (readerOrds[doc]&0xFF);
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc]&0xFF;
- ords[slot] = ord;
- if (ord == 0) {
- values[slot] = null;
- } else {
- assert ord > 0;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.lookup(ord, values[slot]);
- }
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width of doc->ord is 16:
- private final class ShortOrdComparator extends PerSegmentComparator {
- private final short[] readerOrds;
- private final DocTermsIndex termsIndex;
- private final int docBase;
-
- public ShortOrdComparator(short[] readerOrds, DocTermsIndex termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = (readerOrds[doc]&0xFFFF);
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc]&0xFFFF;
- ords[slot] = ord;
- if (ord == 0) {
- values[slot] = null;
- } else {
- assert ord > 0;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.lookup(ord, values[slot]);
- }
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width of doc->ord is 32:
- private final class IntOrdComparator extends PerSegmentComparator {
- private final int[] readerOrds;
- private final DocTermsIndex termsIndex;
- private final int docBase;
-
- public IntOrdComparator(int[] readerOrds, DocTermsIndex termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = readerOrds[doc];
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc];
- ords[slot] = ord;
- if (ord == 0) {
- values[slot] = null;
- } else {
- assert ord > 0;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.lookup(ord, values[slot]);
- }
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width is not a native array
- // size (8, 16, 32):
+ // Used per-segment when docToOrd is null:
private final class AnyOrdComparator extends PerSegmentComparator {
- private final PackedInts.Reader readerOrds;
- private final DocTermsIndex termsIndex;
+ private final SortedDocValues termsIndex;
private final int docBase;
- public AnyOrdComparator(PackedInts.Reader readerOrds, DocTermsIndex termsIndex, int docBase) {
- this.readerOrds = readerOrds;
+ public AnyOrdComparator(SortedDocValues termsIndex, int docBase) {
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@@ -1371,7 +1070,7 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
- final int docOrd = (int) readerOrds.get(doc);
+ final int docOrd = termsIndex.getOrd(doc);
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
@@ -1387,16 +1086,16 @@ public abstract class FieldComparator<T>
@Override
public void copy(int slot, int doc) {
- final int ord = (int) readerOrds.get(doc);
+ final int ord = termsIndex.getOrd(doc);
ords[slot] = ord;
- if (ord == 0) {
+ if (ord == -1) {
values[slot] = null;
} else {
- assert ord > 0;
+ assert ord >= 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
- termsIndex.lookup(ord, values[slot]);
+ termsIndex.lookupOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
@@ -1406,25 +1105,7 @@ public abstract class FieldComparator<T>
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
final int docBase = context.docBase;
termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
- final PackedInts.Reader docToOrd = termsIndex.getDocToOrd();
- FieldComparator<BytesRef> perSegComp = null;
- if (docToOrd.hasArray()) {
- final Object arr = docToOrd.getArray();
- if (arr instanceof byte[]) {
- perSegComp = new ByteOrdComparator((byte[]) arr, termsIndex, docBase);
- } else if (arr instanceof short[]) {
- perSegComp = new ShortOrdComparator((short[]) arr, termsIndex, docBase);
- } else if (arr instanceof int[]) {
- perSegComp = new IntOrdComparator((int[]) arr, termsIndex, docBase);
- }
- // Don't specialize the long[] case since it's not
- // possible, ie, worse case is MAX_INT-1 docs with
- // every one having a unique value.
- }
- if (perSegComp == null) {
- perSegComp = new AnyOrdComparator(docToOrd, termsIndex, docBase);
- }
-
+ FieldComparator<BytesRef> perSegComp = new AnyOrdComparator(termsIndex, docBase);
currentReaderGen++;
if (bottomSlot != -1) {
perSegComp.setBottom(bottomSlot);
@@ -1443,13 +1124,13 @@ public abstract class FieldComparator<T>
bottomSameReader = true;
} else {
if (bottomValue == null) {
- // 0 ord is null for all segments
- assert ords[bottomSlot] == 0;
- bottomOrd = 0;
+ // -1 ord is null for all segments
+ assert ords[bottomSlot] == -1;
+ bottomOrd = -1;
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
} else {
- final int index = binarySearch(tempBR, termsIndex, bottomValue);
+ final int index = termsIndex.lookupTerm(bottomValue);
if (index < 0) {
bottomOrd = -index - 2;
bottomSameReader = false;
@@ -1470,449 +1151,14 @@ public abstract class FieldComparator<T>
}
}
- /** Sorts by field's natural Term sort order, using
- * ordinals; this is just like {@link
- * org.apache.lucene.search.FieldComparator.TermValComparator} except it uses DocValues to
- * retrieve the sort ords saved during indexing. */
- public static final class TermOrdValDocValuesComparator extends FieldComparator<BytesRef> {
- /* Ords for each slot.
- @lucene.internal */
- final int[] ords;
-
- /* Values for each slot.
- @lucene.internal */
- final BytesRef[] values;
-
- /* Which reader last copied a value into the slot. When
- we compare two slots, we just compare-by-ord if the
- readerGen is the same; else we must compare the
- values (slower).
- @lucene.internal */
- final int[] readerGen;
-
- /* Gen of current reader we are on.
- @lucene.internal */
- int currentReaderGen = -1;
-
- /* Current reader's doc ord/values.
- @lucene.internal */
- DocValues.SortedSource termsIndex;
-
- /* Comparator for comparing by value.
- @lucene.internal */
- Comparator<BytesRef> comp;
-
- private final String field;
-
- /* Bottom slot, or -1 if queue isn't full yet
- @lucene.internal */
- int bottomSlot = -1;
-
- /* Bottom ord (same as ords[bottomSlot] once bottomSlot
- is set). Cached for faster compares.
- @lucene.internal */
- int bottomOrd;
-
- /* True if current bottom slot matches the current
- reader.
- @lucene.internal */
- boolean bottomSameReader;
-
- /* Bottom value (same as values[bottomSlot] once
- bottomSlot is set). Cached for faster compares.
- @lucene.internal */
- BytesRef bottomValue;
-
- /** @lucene.internal */
- final BytesRef tempBR = new BytesRef();
-
- public TermOrdValDocValuesComparator(int numHits, String field) {
- ords = new int[numHits];
- values = new BytesRef[numHits];
- readerGen = new int[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- if (readerGen[slot1] == readerGen[slot2]) {
- return ords[slot1] - ords[slot2];
- }
-
- final BytesRef val1 = values[slot1];
- final BytesRef val2 = values[slot2];
- if (val1 == null) {
- if (val2 == null) {
- return 0;
- }
- return -1;
- } else if (val2 == null) {
- return 1;
- }
- return comp.compare(val1, val2);
- }
-
- @Override
- public int compareBottom(int doc) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void copy(int slot, int doc) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int compareDocToValue(int doc, BytesRef value) {
- return termsIndex.getBytes(doc, tempBR).compareTo(value);
- }
-
- // TODO: would be nice to share these specialized impls
- // w/ TermOrdValComparator
-
- /** Base class for specialized (per bit width of the
- * ords) per-segment comparator. NOTE: this is messy;
- * we do this only because hotspot can't reliably inline
- * the underlying array access when looking up doc->ord
- * @lucene.internal
- */
- abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
-
- @Override
- public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- return TermOrdValDocValuesComparator.this.setNextReader(context);
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- return TermOrdValDocValuesComparator.this.compare(slot1, slot2);
- }
-
- @Override
- public void setBottom(final int bottom) {
- TermOrdValDocValuesComparator.this.setBottom(bottom);
- }
-
- @Override
- public BytesRef value(int slot) {
- return TermOrdValDocValuesComparator.this.value(slot);
- }
-
- @Override
- public int compareValues(BytesRef val1, BytesRef val2) {
- assert val1 != null;
- assert val2 != null;
- return comp.compare(val1, val2);
- }
-
- @Override
- public int compareDocToValue(int doc, BytesRef value) {
- return TermOrdValDocValuesComparator.this.compareDocToValue(doc, value);
- }
- }
-
- // Used per-segment when bit width of doc->ord is 8:
- private final class ByteOrdComparator extends PerSegmentComparator {
- private final byte[] readerOrds;
- private final DocValues.SortedSource termsIndex;
- private final int docBase;
-
- public ByteOrdComparator(byte[] readerOrds, DocValues.SortedSource termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = readerOrds[doc]&0xFF;
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc]&0xFF;
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width of doc->ord is 16:
- private final class ShortOrdComparator extends PerSegmentComparator {
- private final short[] readerOrds;
- private final DocValues.SortedSource termsIndex;
- private final int docBase;
-
- public ShortOrdComparator(short[] readerOrds, DocValues.SortedSource termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = readerOrds[doc]&0xFFFF;
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc]&0xFFFF;
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width of doc->ord is 32:
- private final class IntOrdComparator extends PerSegmentComparator {
- private final int[] readerOrds;
- private final DocValues.SortedSource termsIndex;
- private final int docBase;
-
- public IntOrdComparator(int[] readerOrds, DocValues.SortedSource termsIndex, int docBase) {
- this.readerOrds = readerOrds;
- this.termsIndex = termsIndex;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = readerOrds[doc];
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = readerOrds[doc];
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when bit width is not a native array
- // size (8, 16, 32):
- private final class AnyPackedDocToOrdComparator extends PerSegmentComparator {
- private final PackedInts.Reader readerOrds;
- private final int docBase;
-
- public AnyPackedDocToOrdComparator(PackedInts.Reader readerOrds, int docBase) {
- this.readerOrds = readerOrds;
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottomSlot != -1;
- final int docOrd = (int) readerOrds.get(doc);
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = (int) readerOrds.get(doc);
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- // Used per-segment when DV doesn't use packed ints for
- // docToOrds:
- private final class AnyOrdComparator extends PerSegmentComparator {
- private final int docBase;
-
- public AnyOrdComparator(int docBase) {
- this.docBase = docBase;
- }
-
- @Override
- public int compareBottom(int doc) {
- final int docOrd = termsIndex.ord(doc);
- if (bottomSameReader) {
- // ord is precisely comparable, even in the equal case
- return bottomOrd - docOrd;
- } else if (bottomOrd >= docOrd) {
- // the equals case always means bottom is > doc
- // (because we set bottomOrd to the lower bound in
- // setBottom):
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public void copy(int slot, int doc) {
- final int ord = termsIndex.ord(doc);
- ords[slot] = ord;
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- termsIndex.getByOrd(ord, values[slot]);
- readerGen[slot] = currentReaderGen;
- }
- }
-
- @Override
- public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- final int docBase = context.docBase;
-
- final DocValues dv = context.reader().docValues(field);
- if (dv == null) {
- // This may mean entire segment had no docs with
- // this DV field; use default field value (empty
- // byte[]) in this case:
- termsIndex = DocValues.getDefaultSortedSource(DocValues.Type.BYTES_VAR_SORTED, context.reader().maxDoc());
- } else {
- termsIndex = dv.getSource().asSortedSource();
- if (termsIndex == null) {
- // This means segment has doc values, but they are
- // not able to provide a sorted source; consider
- // this a hard error:
- throw new IllegalStateException("DocValues exist for field \"" + field + "\", but not as a sorted source: type=" + dv.getSource().getType() + " reader=" + context.reader());
- }
- }
-
- comp = termsIndex.getComparator();
-
- FieldComparator<BytesRef> perSegComp = null;
- if (termsIndex.hasPackedDocToOrd()) {
- final PackedInts.Reader docToOrd = termsIndex.getDocToOrd();
- if (docToOrd.hasArray()) {
- final Object arr = docToOrd.getArray();
- assert arr != null;
- if (arr instanceof byte[]) {
- // 8 bit packed
- perSegComp = new ByteOrdComparator((byte[]) arr, termsIndex, docBase);
- } else if (arr instanceof short[]) {
- // 16 bit packed
- perSegComp = new ShortOrdComparator((short[]) arr, termsIndex, docBase);
- } else if (arr instanceof int[]) {
- // 32 bit packed
- perSegComp = new IntOrdComparator((int[]) arr, termsIndex, docBase);
- }
- }
-
- if (perSegComp == null) {
- perSegComp = new AnyPackedDocToOrdComparator(docToOrd, docBase);
- }
- } else {
- if (perSegComp == null) {
- perSegComp = new AnyOrdComparator(docBase);
- }
- }
-
- currentReaderGen++;
- if (bottomSlot != -1) {
- perSegComp.setBottom(bottomSlot);
- }
-
- return perSegComp;
- }
-
- @Override
- public void setBottom(final int bottom) {
- bottomSlot = bottom;
-
- bottomValue = values[bottomSlot];
- if (currentReaderGen == readerGen[bottomSlot]) {
- bottomOrd = ords[bottomSlot];
- bottomSameReader = true;
- } else {
- if (bottomValue == null) {
- // 0 ord is null for all segments
- assert ords[bottomSlot] == 0;
- bottomOrd = 0;
- bottomSameReader = true;
- readerGen[bottomSlot] = currentReaderGen;
- } else {
- final int index = termsIndex.getOrdByValue(bottomValue, tempBR);
- if (index < 0) {
- bottomOrd = -index - 2;
- bottomSameReader = false;
- } else {
- bottomOrd = index;
- // exact value match
- bottomSameReader = true;
- readerGen[bottomSlot] = currentReaderGen;
- ords[bottomSlot] = bottomOrd;
- }
- }
- }
- }
-
- @Override
- public BytesRef value(int slot) {
- return values[slot];
- }
- }
-
- /** Sorts by field's natural Term sort order. All
- * comparisons are done using BytesRef.compareTo, which is
- * slow for medium to large result sets but possibly
- * very fast for very small results sets. */
- public static final class TermValComparator extends FieldComparator<BytesRef> {
+ /** Sorts by field's natural Term sort order. All
+ * comparisons are done using BytesRef.compareTo, which is
+ * slow for medium to large result sets but possibly
+ * very fast for very small results sets. */
+ public static final class TermValComparator extends FieldComparator<BytesRef> {
private BytesRef[] values;
- private DocTerms docTerms;
+ private BinaryDocValues docTerms;
private final String field;
private BytesRef bottom;
private final BytesRef tempBR = new BytesRef();
@@ -1940,16 +1186,16 @@ public abstract class FieldComparator<T>
@Override
public int compareBottom(int doc) {
- BytesRef val2 = docTerms.getTerm(doc, tempBR);
- if (bottom == null) {
- if (val2 == null) {
+ docTerms.get(doc, tempBR);
+ if (bottom.bytes == BinaryDocValues.MISSING) {
+ if (tempBR.bytes == BinaryDocValues.MISSING) {
return 0;
}
return -1;
- } else if (val2 == null) {
+ } else if (tempBR.bytes == BinaryDocValues.MISSING) {
return 1;
}
- return bottom.compareTo(val2);
+ return bottom.compareTo(tempBR);
}
@Override
@@ -1957,7 +1203,7 @@ public abstract class FieldComparator<T>
if (values[slot] == null) {
values[slot] = new BytesRef();
}
- docTerms.getTerm(doc, values[slot]);
+ docTerms.get(doc, values[slot]);
}
@Override
@@ -1991,106 +1237,8 @@ public abstract class FieldComparator<T>
@Override
public int compareDocToValue(int doc, BytesRef value) {
- return docTerms.getTerm(doc, tempBR).compareTo(value);
- }
- }
-
- /** Sorts by field's natural Term sort order. All
- * comparisons are done using BytesRef.compareTo, which is
- * slow for medium to large result sets but possibly
- * very fast for very small results sets. The BytesRef
- * values are obtained using {@link AtomicReader#docValues}. */
- public static final class TermValDocValuesComparator extends FieldComparator<BytesRef> {
-
- private BytesRef[] values;
- private DocValues.Source docTerms;
- private final String field;
- private BytesRef bottom;
- private final BytesRef tempBR = new BytesRef();
-
- TermValDocValuesComparator(int numHits, String field) {
- values = new BytesRef[numHits];
- this.field = field;
- }
-
- @Override
- public int compare(int slot1, int slot2) {
- assert values[slot1] != null;
- assert values[slot2] != null;
- return values[slot1].compareTo(values[slot2]);
- }
-
- @Override
- public int compareBottom(int doc) {
- assert bottom != null;
- return bottom.compareTo(docTerms.getBytes(doc, tempBR));
- }
-
- @Override
- public void copy(int slot, int doc) {
- if (values[slot] == null) {
- values[slot] = new BytesRef();
- }
- docTerms.getBytes(doc, values[slot]);
- }
-
- @Override
- public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
- final DocValues dv = context.reader().docValues(field);
- if (dv != null) {
- docTerms = dv.getSource();
- } else {
- docTerms = DocValues.getDefaultSource(DocValues.Type.BYTES_VAR_DEREF);
- }
- return this;
- }
-
- @Override
- public void setBottom(final int bottom) {
- this.bottom = values[bottom];
- }
-
- @Override
- public BytesRef value(int slot) {
- return values[slot];
- }
-
- @Override
- public int compareValues(BytesRef val1, BytesRef val2) {
- assert val1 != null;
- assert val2 != null;
- return val1.compareTo(val2);
- }
-
- @Override
- public int compareDocToValue(int doc, BytesRef value) {
- return docTerms.getBytes(doc, tempBR).compareTo(value);
- }
- }
-
- final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key) {
- return binarySearch(br, a, key, 1, a.numOrd()-1);
- }
-
- final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key, int low, int high) {
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- BytesRef midVal = a.lookup(mid, br);
- int cmp;
- if (midVal != null) {
- cmp = midVal.compareTo(key);
- } else {
- cmp = -1;
- }
-
- if (cmp < 0)
- low = mid + 1;
- else if (cmp > 0)
- high = mid - 1;
- else
- return mid;
+ docTerms.get(doc, tempBR);
+ return tempBR.compareTo(value);
}
- return -(low + 1);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/SortField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/SortField.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/SortField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/SortField.java Thu Feb 7 20:48:21 2013
@@ -257,7 +257,6 @@ public class SortField {
@Override
public String toString() {
StringBuilder buffer = new StringBuilder();
- String dv = useIndexValues ? " [dv]" : "";
switch (type) {
case SCORE:
buffer.append("<score>");
@@ -268,11 +267,11 @@ public class SortField {
break;
case STRING:
- buffer.append("<string" + dv + ": \"").append(field).append("\">");
+ buffer.append("<string" + ": \"").append(field).append("\">");
break;
case STRING_VAL:
- buffer.append("<string_val" + dv + ": \"").append(field).append("\">");
+ buffer.append("<string_val" + ": \"").append(field).append("\">");
break;
case BYTE:
@@ -284,7 +283,7 @@ public class SortField {
break;
case INT:
- buffer.append("<int" + dv + ": \"").append(field).append("\">");
+ buffer.append("<int" + ": \"").append(field).append("\">");
break;
case LONG:
@@ -292,11 +291,11 @@ public class SortField {
break;
case FLOAT:
- buffer.append("<float" + dv + ": \"").append(field).append("\">");
+ buffer.append("<float" + ": \"").append(field).append("\">");
break;
case DOUBLE:
- buffer.append("<double" + dv + ": \"").append(field).append("\">");
+ buffer.append("<double" + ": \"").append(field).append("\">");
break;
case CUSTOM:
@@ -347,16 +346,6 @@ public class SortField {
return hash;
}
- private boolean useIndexValues;
-
- public void setUseIndexValues(boolean b) {
- useIndexValues = b;
- }
-
- public boolean getUseIndexValues() {
- return useIndexValues;
- }
-
private Comparator<BytesRef> bytesComparator = BytesRef.getUTF8SortedAsUnicodeComparator();
public void setBytesComparator(Comparator<BytesRef> b) {
@@ -389,18 +378,10 @@ public class SortField {
return new FieldComparator.DocComparator(numHits);
case INT:
- if (useIndexValues) {
- return new FieldComparator.IntDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.IntComparator(numHits, field, parser, (Integer) missingValue);
- }
+ return new FieldComparator.IntComparator(numHits, field, parser, (Integer) missingValue);
case FLOAT:
- if (useIndexValues) {
- return new FieldComparator.FloatDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.FloatComparator(numHits, field, parser, (Float) missingValue);
- }
+ return new FieldComparator.FloatComparator(numHits, field, parser, (Float) missingValue);
case LONG:
return new FieldComparator.LongComparator(numHits, field, parser, (Long) missingValue);
@@ -419,18 +400,10 @@ public class SortField {
return comparatorSource.newComparator(field, numHits, sortPos, reverse);
case STRING:
- if (useIndexValues) {
- return new FieldComparator.TermOrdValDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.TermOrdValComparator(numHits, field);
- }
+ return new FieldComparator.TermOrdValComparator(numHits, field);
case STRING_VAL:
- if (useIndexValues) {
- return new FieldComparator.TermValDocValuesComparator(numHits, field);
- } else {
- return new FieldComparator.TermValComparator(numHits, field);
- }
+ return new FieldComparator.TermValComparator(numHits, field);
case REWRITEABLE:
throw new IllegalStateException("SortField needs to be rewritten through Sort.rewrite(..) and SortField.rewrite(..)");
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/package.html Thu Feb 7 20:48:21 2013
@@ -338,7 +338,7 @@ extend by plugging in a different compon
Finally, you can extend the low level {@link org.apache.lucene.search.similarities.Similarity Similarity} directly
to implement a new retrieval model, or to use external scoring factors particular to your application. For example,
a custom Similarity can access per-document values via {@link org.apache.lucene.search.FieldCache FieldCache} or
-{@link org.apache.lucene.index.DocValues} and integrate them into the score.
+{@link org.apache.lucene.index.NumericDocValues} and integrate them into the score.
</p>
<p>
See the {@link org.apache.lucene.search.similarities} package documentation for information
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Thu Feb 7 20:48:21 2013
@@ -20,9 +20,8 @@ package org.apache.lucene.search.similar
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -136,9 +135,9 @@ public class BM25Similarity extends Simi
@Override
- public final void computeNorm(FieldInvertState state, Norm norm) {
+ public final long computeNorm(FieldInvertState state) {
final int numTerms = discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength();
- norm.setByte(encodeNormValue(state.getBoost(), numTerms));
+ return encodeNormValue(state.getBoost(), numTerms);
}
/**
@@ -215,7 +214,7 @@ public class BM25Similarity extends Simi
@Override
public final ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
BM25Stats bm25stats = (BM25Stats) stats;
- final DocValues norms = context.reader().normValues(bm25stats.field);
+ final NumericDocValues norms = context.reader().getNormValues(bm25stats.field);
return norms == null
? new ExactBM25DocScorerNoNorms(bm25stats)
: new ExactBM25DocScorer(bm25stats, norms);
@@ -224,26 +223,26 @@ public class BM25Similarity extends Simi
@Override
public final SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
BM25Stats bm25stats = (BM25Stats) stats;
- return new SloppyBM25DocScorer(bm25stats, context.reader().normValues(bm25stats.field));
+ return new SloppyBM25DocScorer(bm25stats, context.reader().getNormValues(bm25stats.field));
}
private class ExactBM25DocScorer extends ExactSimScorer {
private final BM25Stats stats;
private final float weightValue;
- private final byte[] norms;
+ private final NumericDocValues norms;
private final float[] cache;
- ExactBM25DocScorer(BM25Stats stats, DocValues norms) throws IOException {
+ ExactBM25DocScorer(BM25Stats stats, NumericDocValues norms) throws IOException {
assert norms != null;
this.stats = stats;
this.weightValue = stats.weight * (k1 + 1); // boost * idf * (k1 + 1)
this.cache = stats.cache;
- this.norms = (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, int freq) {
- return weightValue * freq / (freq + cache[norms[doc] & 0xFF]);
+ return weightValue * freq / (freq + cache[(byte)norms.get(doc) & 0xFF]);
}
@Override
@@ -283,20 +282,20 @@ public class BM25Similarity extends Simi
private class SloppyBM25DocScorer extends SloppySimScorer {
private final BM25Stats stats;
private final float weightValue; // boost * idf * (k1 + 1)
- private final byte[] norms;
+ private final NumericDocValues norms;
private final float[] cache;
- SloppyBM25DocScorer(BM25Stats stats, DocValues norms) throws IOException {
+ SloppyBM25DocScorer(BM25Stats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.weight * (k1 + 1);
this.cache = stats.cache;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, float freq) {
// if there are no norms, we act as if b=0
- float norm = norms == null ? k1 : cache[norms[doc] & 0xFF];
+ float norm = norms == null ? k1 : cache[(byte)norms.get(doc) & 0xFF];
return weightValue * freq / (freq + norm);
}
@@ -356,7 +355,7 @@ public class BM25Similarity extends Simi
}
}
- private Explanation explainScore(int doc, Explanation freq, BM25Stats stats, byte[] norms) {
+ private Explanation explainScore(int doc, Explanation freq, BM25Stats stats, NumericDocValues norms) {
Explanation result = new Explanation();
result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
@@ -374,7 +373,7 @@ public class BM25Similarity extends Simi
tfNormExpl.addDetail(new Explanation(0, "parameter b (norms omitted for field)"));
tfNormExpl.setValue((freq.getValue() * (k1 + 1)) / (freq.getValue() + k1));
} else {
- float doclen = decodeNormValue(norms[doc]);
+ float doclen = decodeNormValue((byte)norms.get(doc));
tfNormExpl.addDetail(new Explanation(b, "parameter b"));
tfNormExpl.addDetail(new Explanation(stats.avgdl, "avgFieldLength"));
tfNormExpl.addDetail(new Explanation(doclen, "fieldLength"));
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java Thu Feb 7 20:48:21 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.search.similar
*/
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.util.BytesRef;
/** Expert: Default scoring implementation. */
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java Thu Feb 7 20:48:21 2013
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -44,8 +43,8 @@ public class MultiSimilarity extends Sim
}
@Override
- public void computeNorm(FieldInvertState state, Norm norm) {
- sims[0].computeNorm(state, norm);
+ public long computeNorm(FieldInvertState state) {
+ return sims[0].computeNorm(state);
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java Thu Feb 7 20:48:21 2013
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
@@ -42,8 +41,8 @@ public abstract class PerFieldSimilarity
public PerFieldSimilarityWrapper() {}
@Override
- public final void computeNorm(FieldInvertState state, Norm norm) {
- get(state.getName()).computeNorm(state, norm);
+ public final long computeNorm(FieldInvertState state) {
+ return get(state.getName()).computeNorm(state);
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java Thu Feb 7 20:48:21 2013
@@ -19,12 +19,9 @@ package org.apache.lucene.search.similar
import java.io.IOException;
-import org.apache.lucene.document.ByteDocValuesField; // javadoc
-import org.apache.lucene.document.FloatDocValuesField; // javadoc
import org.apache.lucene.index.AtomicReader; // javadoc
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
@@ -52,9 +49,9 @@ import org.apache.lucene.util.SmallFloat
* <a href="#querytime">query-time</a>.
* <p>
* <a name="indextime"/>
- * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState, Norm)}, allowing
+ * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing
* the Similarity implementation to set a per-document value for the field that will
- * be later accessible via {@link AtomicReader#normValues(String)}. Lucene makes no assumption
+ * be later accessible via {@link AtomicReader#getNormValues(String)}. Lucene makes no assumption
* about what is in this norm, but it is most useful for encoding length normalization
* information.
* <p>
@@ -69,9 +66,8 @@ import org.apache.lucene.util.SmallFloat
* depending upon whether the average should reflect field sparsity.
* <p>
* Additional scoring factors can be stored in named
- * <code>*DocValuesField</code>s (such as {@link
- * ByteDocValuesField} or {@link FloatDocValuesField}), and accessed
- * at query-time with {@link AtomicReader#docValues(String)}.
+ * <code>NumericDocValuesField</code>s and accessed
+ * at query-time with {@link AtomicReader#getNumericDocValues(String)}.
* <p>
* Finally, using index-time boosts (either via folding into the normalization byte or
* via DocValues), is an inefficient way to boost the scores of different fields if the
@@ -149,9 +145,6 @@ public abstract class Similarity {
/**
* Computes the normalization value for a field, given the accumulated
* state of term processing for this field (see {@link FieldInvertState}).
- *
- * <p>Implementations should calculate a norm value based on the field
- * state and set that value to the given {@link Norm}.
*
* <p>Matches in longer fields are less precise, so implementations of this
* method usually set smaller values when <code>state.getLength()</code> is large,
@@ -160,10 +153,10 @@ public abstract class Similarity {
* @lucene.experimental
*
* @param state current processing state for this field
- * @param norm holds the computed norm value when this method returns
+ * @return computed norm value
*/
- public abstract void computeNorm(FieldInvertState state, Norm norm);
-
+ public abstract long computeNorm(FieldInvertState state);
+
/**
* Compute any collection-level weight (e.g. IDF, average document length, etc) needed for scoring a query.
*
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Thu Feb 7 20:48:21 2013
@@ -20,9 +20,8 @@ package org.apache.lucene.search.similar
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -199,12 +198,12 @@ public abstract class SimilarityBase ext
ExactSimScorer subScorers[] = new ExactSimScorer[subStats.length];
for (int i = 0; i < subScorers.length; i++) {
BasicStats basicstats = (BasicStats) subStats[i];
- subScorers[i] = new BasicExactDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ subScorers[i] = new BasicExactDocScorer(basicstats, context.reader().getNormValues(basicstats.field));
}
return new MultiSimilarity.MultiExactDocScorer(subScorers);
} else {
BasicStats basicstats = (BasicStats) stats;
- return new BasicExactDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ return new BasicExactDocScorer(basicstats, context.reader().getNormValues(basicstats.field));
}
}
@@ -217,12 +216,12 @@ public abstract class SimilarityBase ext
SloppySimScorer subScorers[] = new SloppySimScorer[subStats.length];
for (int i = 0; i < subScorers.length; i++) {
BasicStats basicstats = (BasicStats) subStats[i];
- subScorers[i] = new BasicSloppyDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ subScorers[i] = new BasicSloppyDocScorer(basicstats, context.reader().getNormValues(basicstats.field));
}
return new MultiSimilarity.MultiSloppyDocScorer(subScorers);
} else {
BasicStats basicstats = (BasicStats) stats;
- return new BasicSloppyDocScorer(basicstats, context.reader().normValues(basicstats.field));
+ return new BasicSloppyDocScorer(basicstats, context.reader().getNormValues(basicstats.field));
}
}
@@ -247,13 +246,13 @@ public abstract class SimilarityBase ext
/** Encodes the document length in the same way as {@link TFIDFSimilarity}. */
@Override
- public void computeNorm(FieldInvertState state, Norm norm) {
+ public long computeNorm(FieldInvertState state) {
final float numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength() / state.getBoost();
- norm.setByte(encodeNormValue(state.getBoost(), numTerms));
+ return encodeNormValue(state.getBoost(), numTerms);
}
/** Decodes a normalization factor (document length) stored in an index.
@@ -286,24 +285,24 @@ public abstract class SimilarityBase ext
*/
private class BasicExactDocScorer extends ExactSimScorer {
private final BasicStats stats;
- private final byte[] norms;
+ private final NumericDocValues norms;
- BasicExactDocScorer(BasicStats stats, DocValues norms) throws IOException {
+ BasicExactDocScorer(BasicStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, int freq) {
// We have to supply something in case norms are omitted
return SimilarityBase.this.score(stats, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
@Override
public Explanation explain(int doc, Explanation freq) {
return SimilarityBase.this.explain(stats, doc, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
}
@@ -315,23 +314,23 @@ public abstract class SimilarityBase ext
*/
private class BasicSloppyDocScorer extends SloppySimScorer {
private final BasicStats stats;
- private final byte[] norms;
+ private final NumericDocValues norms;
- BasicSloppyDocScorer(BasicStats stats, DocValues norms) throws IOException {
+ BasicSloppyDocScorer(BasicStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, float freq) {
// We have to supply something in case norms are omitted
return SimilarityBase.this.score(stats, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
@Override
public Explanation explain(int doc, Explanation freq) {
return SimilarityBase.this.explain(stats, doc, freq,
- norms == null ? 1F : decodeNormValue(norms[doc]));
+ norms == null ? 1F : decodeNormValue((byte)norms.get(doc)));
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java Thu Feb 7 20:48:21 2013
@@ -17,13 +17,11 @@ package org.apache.lucene.search.similar
* limitations under the License.
*/
-
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.Norm;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
@@ -682,17 +680,18 @@ public abstract class TFIDFSimilarity ex
public abstract float lengthNorm(FieldInvertState state);
@Override
- public final void computeNorm(FieldInvertState state, Norm norm) {
+ public final long computeNorm(FieldInvertState state) {
float normValue = lengthNorm(state);
- norm.setByte(encodeNormValue(normValue));
+ return encodeNormValue(normValue);
}
/** Cache of decoded bytes. */
private static final float[] NORM_TABLE = new float[256];
static {
- for (int i = 0; i < 256; i++)
+ for (int i = 0; i < 256; i++) {
NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i);
+ }
}
/** Decodes a normalization factor stored in an index.
@@ -758,13 +757,13 @@ public abstract class TFIDFSimilarity ex
@Override
public final ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
IDFStats idfstats = (IDFStats) stats;
- return new ExactTFIDFDocScorer(idfstats, context.reader().normValues(idfstats.field));
+ return new ExactTFIDFDocScorer(idfstats, context.reader().getNormValues(idfstats.field));
}
@Override
public final SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
IDFStats idfstats = (IDFStats) stats;
- return new SloppyTFIDFDocScorer(idfstats, context.reader().normValues(idfstats.field));
+ return new SloppyTFIDFDocScorer(idfstats, context.reader().getNormValues(idfstats.field));
}
// TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot.
@@ -772,26 +771,19 @@ public abstract class TFIDFSimilarity ex
private final class ExactTFIDFDocScorer extends ExactSimScorer {
private final IDFStats stats;
private final float weightValue;
- private final byte[] norms;
- private static final int SCORE_CACHE_SIZE = 32;
- private float[] scoreCache = new float[SCORE_CACHE_SIZE];
+ private final NumericDocValues norms;
- ExactTFIDFDocScorer(IDFStats stats, DocValues norms) throws IOException {
+ ExactTFIDFDocScorer(IDFStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.value;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
- for (int i = 0; i < SCORE_CACHE_SIZE; i++)
- scoreCache[i] = tf(i) * weightValue;
+ this.norms = norms;
}
@Override
public float score(int doc, int freq) {
- final float raw = // compute tf(f)*weight
- freq < SCORE_CACHE_SIZE // check cache
- ? scoreCache[freq] // cache hit
- : tf(freq)*weightValue; // cache miss
+ final float raw = tf(freq)*weightValue; // compute tf(f)*weight
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
+ return norms == null ? raw : raw * decodeNormValue((byte)norms.get(doc)); // normalize for field
}
@Override
@@ -803,19 +795,19 @@ public abstract class TFIDFSimilarity ex
private final class SloppyTFIDFDocScorer extends SloppySimScorer {
private final IDFStats stats;
private final float weightValue;
- private final byte[] norms;
+ private final NumericDocValues norms;
- SloppyTFIDFDocScorer(IDFStats stats, DocValues norms) throws IOException {
+ SloppyTFIDFDocScorer(IDFStats stats, NumericDocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.value;
- this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
+ this.norms = norms;
}
@Override
public float score(int doc, float freq) {
final float raw = tf(freq) * weightValue; // compute tf(f)*weight
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
+ return norms == null ? raw : raw * decodeNormValue((byte)norms.get(doc)); // normalize for field
}
@Override
@@ -865,9 +857,9 @@ public abstract class TFIDFSimilarity ex
queryWeight *= this.queryNorm; // normalize query weight
value = queryWeight * idf.getValue(); // idf for document
}
- }
-
- private Explanation explainScore(int doc, Explanation freq, IDFStats stats, byte[] norms) {
+ }
+
+ private Explanation explainScore(int doc, Explanation freq, IDFStats stats, NumericDocValues norms) {
Explanation result = new Explanation();
result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
@@ -903,7 +895,7 @@ public abstract class TFIDFSimilarity ex
Explanation fieldNormExpl = new Explanation();
float fieldNorm =
- norms!=null ? decodeNormValue(norms[doc]) : 1.0f;
+ norms!=null ? decodeNormValue((byte) norms.get(doc)) : 1.0f;
fieldNormExpl.setValue(fieldNorm);
fieldNormExpl.setDescription("fieldNorm(doc="+doc+")");
fieldExpl.addDetail(fieldNormExpl);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/package.html?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/similarities/package.html Thu Feb 7 20:48:21 2013
@@ -132,7 +132,7 @@ subclassing the Similarity, one can simp
matching term occurs. In these
cases people have overridden Similarity to return 1 from the tf() method.</p></li>
<li><p>Changing Length Normalization — By overriding
- {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state, Norm)},
+ {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state)},
it is possible to discount how the length of a field contributes
to a score. In {@link org.apache.lucene.search.similarities.DefaultSimilarity},
lengthNorm = 1 / (numTerms in field)^0.5, but if one changes this to be
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java Thu Feb 7 20:48:21 2013
@@ -220,6 +220,7 @@ final class CompoundFileWriter implement
assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written";
seenIDs.add(id);
final DirectCFSIndexOutput out;
+
if ((outputLocked = outputTaken.compareAndSet(false, true))) {
out = new DirectCFSIndexOutput(getOutput(), entry, false);
} else {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java Thu Feb 7 20:48:21 2013
@@ -16,12 +16,10 @@ package org.apache.lucene.util;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import java.io.IOException;
+
import java.util.Arrays;
import java.util.List;
-import org.apache.lucene.store.DataOutput;
-
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
/**
@@ -255,8 +253,9 @@ public final class ByteBlockPool {
final int newSize = LEVEL_SIZE_ARRAY[newLevel];
// Maybe allocate another block
- if (byteUpto > BYTE_BLOCK_SIZE-newSize)
+ if (byteUpto > BYTE_BLOCK_SIZE-newSize) {
nextBuffer();
+ }
final int newUpto = byteUpto;
final int offset = newUpto + byteOffset;
@@ -282,7 +281,7 @@ public final class ByteBlockPool {
// Fill in a BytesRef from term's length & bytes encoded in
// byte block
- public final BytesRef setBytesRef(BytesRef term, int textStart) {
+ public void setBytesRef(BytesRef term, int textStart) {
final byte[] bytes = term.bytes = buffers[textStart >> BYTE_BLOCK_SHIFT];
int pos = textStart & BYTE_BLOCK_MASK;
if ((bytes[pos] & 0x80) == 0) {
@@ -295,27 +294,17 @@ public final class ByteBlockPool {
term.offset = pos+2;
}
assert term.length >= 0;
- return term;
- }
- /**
- * Dereferences the byte block according to {@link BytesRef} offset. The offset
- * is interpreted as the absolute offset into the {@link ByteBlockPool}.
- */
- public final BytesRef deref(BytesRef bytes) {
- final int offset = bytes.offset;
- byte[] buffer = buffers[offset >> BYTE_BLOCK_SHIFT];
- int pos = offset & BYTE_BLOCK_MASK;
- bytes.bytes = buffer;
- bytes.offset = pos;
- return bytes;
}
/**
- * Copies the given {@link BytesRef} at the current positions (
- * {@link #byteUpto} across buffer boundaries
+ * Appends the bytes in the provided {@link BytesRef} at
+ * the current position.
*/
- public final void copy(final BytesRef bytes) {
+ public void append(final BytesRef bytes) {
int length = bytes.length;
+ if (length == 0) {
+ return;
+ }
int offset = bytes.offset;
int overflow = (length + byteUpto) - BYTE_BLOCK_SIZE;
do {
@@ -325,9 +314,11 @@ public final class ByteBlockPool {
break;
} else {
final int bytesToCopy = length-overflow;
- System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy);
- offset += bytesToCopy;
- length -= bytesToCopy;
+ if (bytesToCopy > 0) {
+ System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy);
+ offset += bytesToCopy;
+ length -= bytesToCopy;
+ }
nextBuffer();
overflow = overflow - BYTE_BLOCK_SIZE;
}
@@ -335,48 +326,34 @@ public final class ByteBlockPool {
}
/**
- * Copies bytes from the pool starting at the given offset with the given
- * length into the given {@link BytesRef} at offset <tt>0</tt> and returns it.
+ * Reads bytes bytes out of the pool starting at the given offset with the given
+ * length into the given byte array at offset <tt>off</tt>.
* <p>Note: this method allows to copy across block boundaries.</p>
*/
- public final BytesRef copyFrom(final BytesRef bytes, final int offset, final int length) {
- bytes.offset = 0;
- bytes.grow(length);
- bytes.length = length;
- int bufferIndex = offset >> BYTE_BLOCK_SHIFT;
+ public void readBytes(final long offset, final byte bytes[], final int off, final int length) {
+ if (length == 0) {
+ return;
+ }
+ int bytesOffset = off;
+ int bytesLength = length;
+ int bufferIndex = (int) (offset >> BYTE_BLOCK_SHIFT);
byte[] buffer = buffers[bufferIndex];
- int pos = offset & BYTE_BLOCK_MASK;
+ int pos = (int) (offset & BYTE_BLOCK_MASK);
int overflow = (pos + length) - BYTE_BLOCK_SIZE;
do {
if (overflow <= 0) {
- System.arraycopy(buffer, pos, bytes.bytes, bytes.offset, bytes.length);
- bytes.length = length;
- bytes.offset = 0;
+ System.arraycopy(buffer, pos, bytes, bytesOffset, bytesLength);
break;
} else {
final int bytesToCopy = length - overflow;
- System.arraycopy(buffer, pos, bytes.bytes, bytes.offset, bytesToCopy);
+ System.arraycopy(buffer, pos, bytes, bytesOffset, bytesToCopy);
pos = 0;
- bytes.length -= bytesToCopy;
- bytes.offset += bytesToCopy;
+ bytesLength -= bytesToCopy;
+ bytesOffset += bytesToCopy;
buffer = buffers[++bufferIndex];
overflow = overflow - BYTE_BLOCK_SIZE;
}
} while (true);
- return bytes;
- }
-
- /**
- * Writes the pools content to the given {@link DataOutput}
- */
- public final void writePool(final DataOutput out) throws IOException {
- int bytesOffset = byteOffset;
- int block = 0;
- while (bytesOffset > 0) {
- out.writeBytes(buffers[block++], BYTE_BLOCK_SIZE);
- bytesOffset -= BYTE_BLOCK_SIZE;
- }
- out.writeBytes(buffers[block], byteUpto);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRef.java?rev=1443717&r1=1443716&r2=1443717&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRef.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRef.java Thu Feb 7 20:48:21 2013
@@ -52,13 +52,10 @@ public final class BytesRef implements C
* bytes should not be null.
*/
public BytesRef(byte[] bytes, int offset, int length) {
- assert bytes != null;
- assert offset >= 0;
- assert length >= 0;
- assert bytes.length >= offset + length;
this.bytes = bytes;
this.offset = offset;
this.length = length;
+ assert isValid();
}
/** This instance will directly reference bytes w/o making a copy.
@@ -340,4 +337,33 @@ public final class BytesRef implements C
copy.copyBytes(other);
return copy;
}
+
+ /**
+ * Performs internal consistency checks.
+ * Always returns true (or throws IllegalStateException)
+ */
+ public boolean isValid() {
+ if (bytes == null) {
+ throw new IllegalStateException("bytes is null");
+ }
+ if (length < 0) {
+ throw new IllegalStateException("length is negative: " + length);
+ }
+ if (length > bytes.length) {
+ throw new IllegalStateException("length is out of bounds: " + length + ",bytes.length=" + bytes.length);
+ }
+ if (offset < 0) {
+ throw new IllegalStateException("offset is negative: " + offset);
+ }
+ if (offset > bytes.length) {
+ throw new IllegalStateException("offset out of bounds: " + offset + ",bytes.length=" + bytes.length);
+ }
+ if (offset + length < 0) {
+ throw new IllegalStateException("offset+length is negative: offset=" + offset + ",length=" + length);
+ }
+ if (offset + length > bytes.length) {
+ throw new IllegalStateException("offset+length out of bounds: offset=" + offset + ",length=" + length + ",bytes.length=" + bytes.length);
+ }
+ return true;
+ }
}