You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/01/20 00:20:43 UTC
svn commit: r1061060 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/analysis/NumericTokenStream.java
test/org/apache/lucene/analysis/TestNumericTokenStream.java
Author: uschindler
Date: Wed Jan 19 23:20:43 2011
New Revision: 1061060
URL: http://svn.apache.org/viewvc?rev=1061060&view=rev
Log:
LUCENE-2875: Rework token metadata for NumericTermAttribute
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java?rev=1061060&r1=1061059&r2=1061060&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/NumericTokenStream.java Wed Jan 19 23:20:43 2011
@@ -96,23 +96,28 @@ public final class NumericTokenStream ex
/** The lower precision tokens gets this token type assigned. */
public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecNumeric";
- /** <b>Expert:</b> Use this attribute to get the details of the currently generated token
+ /** <b>Expert:</b> Use this attribute to get the details of the currently generated token.
* @lucene.experimental
* @since 4.0
*/
public interface NumericTermAttribute extends Attribute {
/** Returns current shift value, undefined before first token */
int getShift();
- /** Returns {@link NumericTokenStream}'s raw value as {@code long} */
+ /** Returns current token's raw value as {@code long} with all {@link #getShift} applied, undefined before first token */
long getRawValue();
/** Returns value size in bits (32 for {@code float}, {@code int}; 64 for {@code double}, {@code long}) */
int getValueSize();
- /** @lucene.internal Don't call this method! */
- void init(long rawValue, int valSize, int precisionStep);
- /** @lucene.internal Don't call this method! */
+ /** <em>Don't call this method!</em>
+ * @lucene.internal */
+ void init(long value, int valSize, int precisionStep, int shift);
+
+ /** <em>Don't call this method!</em>
+ * @lucene.internal */
void setShift(int shift);
- /** @lucene.internal Don't call this method! */
+
+ /** <em>Don't call this method!</em>
+ * @lucene.internal */
int incShift();
}
@@ -132,16 +137,20 @@ public final class NumericTokenStream ex
}
}
+ /** Implementatation of {@link NumericTermAttribute}.
+ * @lucene.internal
+ * @since 4.0
+ */
public static final class NumericTermAttributeImpl extends AttributeImpl implements NumericTermAttribute,TermToBytesRefAttribute {
- private long rawValue = 0L;
+ private long value = 0L;
private int valueSize = 0, shift = 0, precisionStep = 0;
public int toBytesRef(BytesRef bytes) {
try {
assert valueSize == 64 || valueSize == 32;
return (valueSize == 64) ?
- NumericUtils.longToPrefixCoded(rawValue, shift, bytes) :
- NumericUtils.intToPrefixCoded((int) rawValue, shift, bytes);
+ NumericUtils.longToPrefixCoded(value, shift, bytes) :
+ NumericUtils.intToPrefixCoded((int) value, shift, bytes);
} catch (IllegalArgumentException iae) {
// return empty token before first or after last
bytes.length = 0;
@@ -155,13 +164,14 @@ public final class NumericTokenStream ex
return (shift += precisionStep);
}
- public long getRawValue() { return rawValue; }
+ public long getRawValue() { return value & ~((1L << shift) - 1L); }
public int getValueSize() { return valueSize; }
- public void init(long rawValue, int valueSize, int precisionStep) {
- this.rawValue = rawValue;
+ public void init(long value, int valueSize, int precisionStep, int shift) {
+ this.value = value;
this.valueSize = valueSize;
this.precisionStep = precisionStep;
+ this.shift = shift;
}
@Override
@@ -176,16 +186,14 @@ public final class NumericTokenStream ex
toBytesRef(bytes);
reflector.reflect(TermToBytesRefAttribute.class, "bytes", bytes);
reflector.reflect(NumericTermAttribute.class, "shift", shift);
- reflector.reflect(NumericTermAttribute.class, "rawValue", rawValue);
+ reflector.reflect(NumericTermAttribute.class, "rawValue", getRawValue());
reflector.reflect(NumericTermAttribute.class, "valueSize", valueSize);
- reflector.reflect(NumericTermAttribute.class, "precisionStep", precisionStep);
}
@Override
public void copyTo(AttributeImpl target) {
final NumericTermAttribute a = (NumericTermAttribute) target;
- a.init(rawValue, valueSize, precisionStep);
- a.setShift(shift);
+ a.init(value, valueSize, precisionStep, shift);
}
}
@@ -229,8 +237,7 @@ public final class NumericTokenStream ex
* <code>new Field(name, new NumericTokenStream(precisionStep).setLongValue(value))</code>
*/
public NumericTokenStream setLongValue(final long value) {
- numericAtt.init(value, valSize = 64, precisionStep);
- numericAtt.setShift(-precisionStep);
+ numericAtt.init(value, valSize = 64, precisionStep, -precisionStep);
return this;
}
@@ -241,8 +248,7 @@ public final class NumericTokenStream ex
* <code>new Field(name, new NumericTokenStream(precisionStep).setIntValue(value))</code>
*/
public NumericTokenStream setIntValue(final int value) {
- numericAtt.init(value, valSize = 32, precisionStep);
- numericAtt.setShift(-precisionStep);
+ numericAtt.init(value, valSize = 32, precisionStep, -precisionStep);
return this;
}
@@ -253,8 +259,7 @@ public final class NumericTokenStream ex
* <code>new Field(name, new NumericTokenStream(precisionStep).setDoubleValue(value))</code>
*/
public NumericTokenStream setDoubleValue(final double value) {
- numericAtt.init(NumericUtils.doubleToSortableLong(value), valSize = 64, precisionStep);
- numericAtt.setShift(-precisionStep);
+ numericAtt.init(NumericUtils.doubleToSortableLong(value), valSize = 64, precisionStep, -precisionStep);
return this;
}
@@ -265,8 +270,7 @@ public final class NumericTokenStream ex
* <code>new Field(name, new NumericTokenStream(precisionStep).setFloatValue(value))</code>
*/
public NumericTokenStream setFloatValue(final float value) {
- numericAtt.init(NumericUtils.floatToSortableInt(value), valSize = 32, precisionStep);
- numericAtt.setShift(-precisionStep);
+ numericAtt.init(NumericUtils.floatToSortableInt(value), valSize = 32, precisionStep, -precisionStep);
return this;
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java?rev=1061060&r1=1061059&r2=1061060&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java Wed Jan 19 23:20:43 2011
@@ -38,13 +38,13 @@ public class TestNumericTokenStream exte
final BytesRef bytes = new BytesRef();
stream.reset();
assertEquals(64, numericAtt.getValueSize());
- assertEquals(lvalue, numericAtt.getRawValue());
for (int shift=0; shift<64; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
final int hash = bytesAtt.toBytesRef(bytes);
assertEquals("Hash incorrect", bytes.hashCode(), hash);
assertEquals("Term is incorrectly encoded", lvalue & ~((1L << shift) - 1L), NumericUtils.prefixCodedToLong(bytes));
+ assertEquals("Term raw value is incorrectly encoded", lvalue & ~((1L << shift) - 1L), numericAtt.getRawValue());
assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("More tokens available", stream.incrementToken());
@@ -61,13 +61,13 @@ public class TestNumericTokenStream exte
final BytesRef bytes = new BytesRef();
stream.reset();
assertEquals(32, numericAtt.getValueSize());
- assertEquals(ivalue, numericAtt.getRawValue());
for (int shift=0; shift<32; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
final int hash = bytesAtt.toBytesRef(bytes);
assertEquals("Hash incorrect", bytes.hashCode(), hash);
assertEquals("Term is incorrectly encoded", ivalue & ~((1 << shift) - 1), NumericUtils.prefixCodedToInt(bytes));
+ assertEquals("Term raw value is incorrectly encoded", ((long) ivalue) & ~((1L << shift) - 1L), numericAtt.getRawValue());
assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("More tokens available", stream.incrementToken());