You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/01/16 14:28:58 UTC
svn commit: r1433930 - in /lucene/dev/branches/lucene_solr_4_1: ./ lucene/
lucene/facet/ lucene/facet/src/java/org/apache/lucene/util/encoding/
lucene/facet/src/test/org/apache/lucene/facet/index/params/
lucene/facet/src/test/org/apache/lucene/util/enc...
Author: shaie
Date: Wed Jan 16 13:28:58 2013
New Revision: 1433930
URL: http://svn.apache.org/viewvc?rev=1433930&view=rev
Log:
LUCENE-4620: inline encoding/decoding
Removed:
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/Vint8Test.java
Modified:
lucene/dev/branches/lucene_solr_4_1/ (props changed)
lucene/dev/branches/lucene_solr_4_1/lucene/ (props changed)
lucene/dev/branches/lucene_solr_4_1/lucene/facet/ (props changed)
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/ChunksIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -49,20 +49,20 @@ import org.apache.lucene.util.IntsRef;
* @lucene.experimental
*/
public abstract class ChunksIntEncoder extends IntEncoder {
-
+
/** Holds the values which must be encoded, outside the indicator. */
protected final IntsRef encodeQueue;
-
+
/** Represents bits flag byte. */
protected int indicator = 0;
-
+
/** Counts the current ordinal of the encoded value. */
protected byte ordinal = 0;
-
+
protected ChunksIntEncoder(int chunkSize) {
encodeQueue = new IntsRef(chunkSize);
}
-
+
/**
* Encodes the values of the current chunk. First it writes the indicator, and
* then it encodes the values outside the indicator.
@@ -76,17 +76,40 @@ public abstract class ChunksIntEncoder e
buf.bytes[buf.length++] = ((byte) indicator);
for (int i = 0; i < encodeQueue.length; i++) {
- VInt8.encode(encodeQueue.ints[i], buf);
+ // it is better if the encoding is inlined like so, and not e.g.
+ // in a utility method
+ int value = encodeQueue.ints[i];
+ if ((value & ~0x7F) == 0) {
+ buf.bytes[buf.length] = (byte) value;
+ buf.length++;
+ } else if ((value & ~0x3FFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 1] = (byte) (value & 0x7F);
+ buf.length += 2;
+ } else if ((value & ~0x1FFFFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 2] = (byte) (value & 0x7F);
+ buf.length += 3;
+ } else if ((value & ~0xFFFFFFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 3] = (byte) (value & 0x7F);
+ buf.length += 4;
+ } else {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 4] = (byte) (value & 0x7F);
+ buf.length += 5;
+ }
}
- reset();
- }
-
- @Override
- protected void reset() {
ordinal = 0;
indicator = 0;
encodeQueue.length = 0;
}
-
+
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntDecoder.java Wed Jan 16 13:28:58 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.util.IntsRef;
*
* @lucene.experimental
*/
-public class DGapIntDecoder extends IntDecoder {
+public final class DGapIntDecoder extends IntDecoder {
private final IntDecoder decoder;
@@ -35,13 +35,8 @@ public class DGapIntDecoder extends IntD
}
@Override
- protected void reset() {
- decoder.reset();
- }
-
- @Override
- protected void doDecode(BytesRef buf, IntsRef values, int upto) {
- decoder.doDecode(buf, values, upto);
+ public void decode(BytesRef buf, IntsRef values) {
+ decoder.decode(buf, values);
int prev = 0;
for (int i = 0; i < values.length; i++) {
values.ints[i] += prev;
@@ -51,7 +46,7 @@ public class DGapIntDecoder extends IntD
@Override
public String toString() {
- return "DGap (" + decoder.toString() + ")";
+ return "DGap(" + decoder.toString() + ")";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/DGapIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -35,7 +35,7 @@ import org.apache.lucene.util.IntsRef;
*
* @lucene.experimental
*/
-public class DGapIntEncoder extends IntEncoderFilter {
+public final class DGapIntEncoder extends IntEncoderFilter {
/** Initializes with the given encoder. */
public DGapIntEncoder(IntEncoder encoder) {
@@ -43,14 +43,15 @@ public class DGapIntEncoder extends IntE
}
@Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ public void encode(IntsRef values, BytesRef buf) {
int prev = 0;
+ int upto = values.offset + values.length;
for (int i = values.offset; i < upto; i++) {
int tmp = values.ints[i];
values.ints[i] -= prev;
prev = tmp;
}
- encoder.doEncode(values, buf, upto);
+ encoder.encode(values, buf);
}
@Override
@@ -60,7 +61,7 @@ public class DGapIntEncoder extends IntE
@Override
public String toString() {
- return "DGap (" + encoder.toString() + ")";
+ return "DGap(" + encoder.toString() + ")";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntDecoder.java Wed Jan 16 13:28:58 2013
@@ -45,10 +45,13 @@ public class EightFlagsIntDecoder extend
}
@Override
- protected void doDecode(BytesRef buf, IntsRef values, int upto) {
- while (buf.offset < upto) {
+ public void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0;
+ int upto = buf.offset + buf.length;
+ int offset = buf.offset;
+ while (offset < upto) {
// read indicator
- int indicator = buf.bytes[buf.offset++] & 0xFF;
+ int indicator = buf.bytes[offset++] & 0xFF;
int ordinal = 0;
int capacityNeeded = values.length + 8;
@@ -59,11 +62,21 @@ public class EightFlagsIntDecoder extend
// process indicator, until we read 8 values, or end-of-buffer
while (ordinal != 8) {
if (DECODE_TABLE[indicator][ordinal++] == 0) {
- if (buf.offset == upto) { // end of buffer
+ if (offset == upto) { // end of buffer
return;
}
- // decode the value from the stream.
- values.ints[values.length++] = VInt8.decode(buf) + 2;
+ // it is better if the decoding is inlined like so, and not e.g.
+ // in a utility method
+ int value = 0;
+ while (true) {
+ byte b = buf.bytes[offset++];
+ if (b >= 0) {
+ values.ints[values.length++] = ((value << 7) | b) + 2;
+ break;
+ } else {
+ value = (value << 7) | (b & 0x7F);
+ }
+ }
} else {
values.ints[values.length++] = 1;
}
@@ -73,7 +86,7 @@ public class EightFlagsIntDecoder extend
@Override
public String toString() {
- return "EightFlags (VInt8)";
+ return "EightFlags(VInt8)";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/EightFlagsIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -59,7 +59,9 @@ public class EightFlagsIntEncoder extend
}
@Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ public void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
+ int upto = values.offset + values.length;
for (int i = values.offset; i < upto; i++) {
int value = values.ints[i];
if (value == 1) {
@@ -88,7 +90,7 @@ public class EightFlagsIntEncoder extend
@Override
public String toString() {
- return "EightFlags (VInt)";
+ return "EightFlags(VInt)";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntDecoder.java Wed Jan 16 13:28:58 2013
@@ -45,10 +45,13 @@ public class FourFlagsIntDecoder extends
}
@Override
- protected void doDecode(BytesRef buf, IntsRef values, int upto) {
- while (buf.offset < upto) {
+ public void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0;
+ int upto = buf.offset + buf.length;
+ int offset = buf.offset;
+ while (offset < upto) {
// read indicator
- int indicator = buf.bytes[buf.offset++] & 0xFF;
+ int indicator = buf.bytes[offset++] & 0xFF;
int ordinal = 0;
int capacityNeeded = values.length + 4;
@@ -59,11 +62,21 @@ public class FourFlagsIntDecoder extends
while (ordinal != 4) {
byte decodeVal = DECODE_TABLE[indicator][ordinal++];
if (decodeVal == 0) {
- if (buf.offset == upto) { // end of buffer
+ if (offset == upto) { // end of buffer
return;
}
- // decode the value from the stream.
- values.ints[values.length++] = VInt8.decode(buf) + 4;
+ // it is better if the decoding is inlined like so, and not e.g.
+ // in a utility method
+ int value = 0;
+ while (true) {
+ byte b = buf.bytes[offset++];
+ if (b >= 0) {
+ values.ints[values.length++] = ((value << 7) | b) + 4;
+ break;
+ } else {
+ value = (value << 7) | (b & 0x7F);
+ }
+ }
} else {
values.ints[values.length++] = decodeVal;
}
@@ -73,7 +86,7 @@ public class FourFlagsIntDecoder extends
@Override
public String toString() {
- return "FourFlags (VInt8)";
+ return "FourFlags(VInt8)";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/FourFlagsIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -65,7 +65,9 @@ public class FourFlagsIntEncoder extends
}
@Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ public void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
+ int upto = values.offset + values.length;
for (int i = values.offset; i < upto; i++) {
int value = values.ints[i];
if (value <= 3) {
@@ -94,7 +96,7 @@ public class FourFlagsIntEncoder extends
@Override
public String toString() {
- return "FourFlags (VInt)";
+ return "FourFlags(VInt)";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntDecoder.java Wed Jan 16 13:28:58 2013
@@ -28,43 +28,9 @@ import org.apache.lucene.util.IntsRef;
public abstract class IntDecoder {
/**
- * Performs the actual decoding. Values should be read from
- * {@link BytesRef#offset} up to {@code upto}. Also, {@code values} offset and
- * length are set to 0 and the encoder is expected to update
- * {@link IntsRef#length}, but not {@link IntsRef#offset}.
- *
- * <p>
- * <b>NOTE:</b> it is ok to use the buffer's offset as the current position in
- * the buffer (and modify it), it will be reset by
- * {@link #decode(BytesRef, IntsRef)}.
- */
- protected abstract void doDecode(BytesRef buf, IntsRef values, int upto);
-
- /**
- * Called before {@link #doDecode(BytesRef, IntsRef, int)} so that decoders
- * can reset their state.
- */
- protected void reset() {
- // do nothing by default
- }
-
- /**
* Decodes the values from the buffer into the given {@link IntsRef}. Note
* that {@code values.offset} and {@code values.length} are set to 0.
*/
- public final void decode(BytesRef buf, IntsRef values) {
- values.offset = values.length = 0; // must do that because we cannot grow() them otherwise
-
- // some decoders may use the buffer's offset as a position index, so save
- // current offset.
- int bufOffset = buf.offset;
-
- reset();
- doDecode(buf, values, buf.offset + buf.length);
- assert values.offset == 0 : "offset should not have been modified by the decoder.";
-
- // fix offset
- buf.offset = bufOffset;
- }
+ public abstract void decode(BytesRef buf, IntsRef values);
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoder.java Wed Jan 16 13:28:58 2013
@@ -32,31 +32,10 @@ public abstract class IntEncoder {
public IntEncoder() {}
/**
- * Performs the actual encoding. Values should be read from
- * {@link IntsRef#offset} up to {@code upto}. Also, it is guaranteed that
- * {@code buf's} offset and length are set to 0 and the encoder is expected to
- * update {@link BytesRef#length}, but not {@link BytesRef#offset}.
- */
- protected abstract void doEncode(IntsRef values, BytesRef buf, int upto);
-
- /**
- * Called before {@link #doEncode(IntsRef, BytesRef, int)} so that encoders
- * can reset their state.
- */
- protected void reset() {
- // do nothing by default
- }
-
- /**
* Encodes the values to the given buffer. Note that the buffer's offset and
* length are set to 0.
*/
- public final void encode(IntsRef values, BytesRef buf) {
- buf.offset = buf.length = 0;
- reset();
- doEncode(values, buf, values.offset + values.length);
- assert buf.offset == 0 : "offset should not have been modified by the encoder.";
- }
+ public abstract void encode(IntsRef values, BytesRef buf);
/**
* Returns an {@link IntDecoder} which can decode the values that were encoded
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/IntEncoderFilter.java Wed Jan 16 13:28:58 2013
@@ -31,9 +31,4 @@ public abstract class IntEncoderFilter e
this.encoder = encoder;
}
- @Override
- public void reset() {
- encoder.reset();
- }
-
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntDecoder.java Wed Jan 16 13:28:58 2013
@@ -42,14 +42,10 @@ public class NOnesIntDecoder extends Fou
}
@Override
- protected void reset() {
+ public void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0;
internalBuffer.length = 0;
- super.reset();
- }
-
- @Override
- protected void doDecode(BytesRef buf, IntsRef values, int upto) {
- super.doDecode(buf, internalBuffer, upto);
+ super.decode(buf, internalBuffer);
if (values.ints.length < internalBuffer.length) {
// need space for internalBuffer.length to internalBuffer.length*N,
// grow mildly at first
@@ -84,7 +80,7 @@ public class NOnesIntDecoder extends Fou
@Override
public String toString() {
- return "NOnes (" + n + ") (" + super.toString() + ")";
+ return "NOnes(" + n + ") (" + super.toString() + ")";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/NOnesIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -65,19 +65,15 @@ public class NOnesIntEncoder extends Fou
}
@Override
- protected void reset() {
+ public void encode(IntsRef values, BytesRef buf) {
internalBuffer.length = 0;
- super.reset();
- }
-
- @Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
// make sure the internal buffer is large enough
if (values.length > internalBuffer.ints.length) {
internalBuffer.grow(values.length);
}
int onesCounter = 0;
+ int upto = values.offset + values.length;
for (int i = values.offset; i < upto; i++) {
int value = values.ints[i];
if (value == 1) {
@@ -102,7 +98,7 @@ public class NOnesIntEncoder extends Fou
--onesCounter;
internalBuffer.ints[internalBuffer.length++] = 1;
}
- super.doEncode(internalBuffer, buf, internalBuffer.length);
+ super.encode(internalBuffer, buf);
}
@Override
@@ -112,7 +108,7 @@ public class NOnesIntEncoder extends Fou
@Override
public String toString() {
- return "NOnes (" + n + ") (" + super.toString() + ")";
+ return "NOnes(" + n + ") (" + super.toString() + ")";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntDecoder.java Wed Jan 16 13:28:58 2013
@@ -1,7 +1,9 @@
package org.apache.lucene.util.encoding;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,19 +27,24 @@ import org.apache.lucene.util.IntsRef;
*
* @lucene.experimental
*/
-public class SimpleIntDecoder extends IntDecoder {
+public final class SimpleIntDecoder extends IntDecoder {
@Override
- protected void doDecode(BytesRef buf, IntsRef values, int upto) {
- while (buf.offset < upto) {
- if (values.length == values.ints.length) {
- values.grow(values.length + 10); // grow by few items, however not too many
- }
+ public void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0;
+ int numValues = buf.length / 4; // every value is 4 bytes
+ if (values.ints.length < numValues) { // offset and length are 0
+ values.ints = new int[ArrayUtil.oversize(numValues, RamUsageEstimator.NUM_BYTES_INT)];
+ }
+
+ int offset = buf.offset;
+ int upto = buf.offset + buf.length;
+ while (offset < upto) {
values.ints[values.length++] =
- ((buf.bytes[buf.offset++] & 0xFF) << 24) |
- ((buf.bytes[buf.offset++] & 0xFF) << 16) |
- ((buf.bytes[buf.offset++] & 0xFF) << 8) |
- (buf.bytes[buf.offset++] & 0xFF);
+ ((buf.bytes[offset++] & 0xFF) << 24) |
+ ((buf.bytes[offset++] & 0xFF) << 16) |
+ ((buf.bytes[offset++] & 0xFF) << 8) |
+ (buf.bytes[offset++] & 0xFF);
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SimpleIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -25,16 +25,18 @@ import org.apache.lucene.util.IntsRef;
*
* @lucene.experimental
*/
-public class SimpleIntEncoder extends IntEncoder {
+public final class SimpleIntEncoder extends IntEncoder {
@Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ public void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
// ensure there's enough room in the buffer
int bytesNeeded = values.length * 4;
if (buf.bytes.length < bytesNeeded) {
buf.grow(bytesNeeded);
}
+ int upto = values.offset + values.length;
for (int i = values.offset; i < upto; i++) {
int value = values.ints[i];
buf.bytes[buf.length++] = (byte) (value >>> 24);
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/SortingIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.util.IntsRef;
*
* @lucene.experimental
*/
-public class SortingIntEncoder extends IntEncoderFilter {
+public final class SortingIntEncoder extends IntEncoderFilter {
/** Initializes with the given encoder. */
public SortingIntEncoder(IntEncoder encoder) {
@@ -36,9 +36,9 @@ public class SortingIntEncoder extends I
}
@Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
- Arrays.sort(values.ints, values.offset, upto);
- encoder.doEncode(values, buf, upto);
+ public void encode(IntsRef values, BytesRef buf) {
+ Arrays.sort(values.ints, values.offset, values.offset + values.length);
+ encoder.encode(values, buf);
}
@Override
@@ -48,7 +48,7 @@ public class SortingIntEncoder extends I
@Override
public String toString() {
- return "Sorting (" + encoder.toString() + ")";
+ return "Sorting(" + encoder.toString() + ")";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/UniqueValuesIntEncoder.java Wed Jan 16 13:28:58 2013
@@ -36,9 +36,10 @@ public final class UniqueValuesIntEncode
}
@Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ public void encode(IntsRef values, BytesRef buf) {
int prev = values.ints[values.offset];
int idx = values.offset + 1;
+ int upto = values.offset + values.length;
for (int i = idx; i < upto; i++) {
if (values.ints[i] != prev) {
values.ints[idx++] = values.ints[i];
@@ -46,7 +47,7 @@ public final class UniqueValuesIntEncode
}
}
values.length = idx - values.offset;
- encoder.doEncode(values, buf, idx);
+ encoder.encode(values, buf);
}
@Override
@@ -56,7 +57,7 @@ public final class UniqueValuesIntEncode
@Override
public String toString() {
- return "Unique (" + encoder.toString() + ")";
+ return "Unique(" + encoder.toString() + ")";
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntDecoder.java Wed Jan 16 13:28:58 2013
@@ -1,7 +1,9 @@
package org.apache.lucene.util.encoding;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,15 +27,32 @@ import org.apache.lucene.util.IntsRef;
*
* @lucene.experimental
*/
-public class VInt8IntDecoder extends IntDecoder {
+public final class VInt8IntDecoder extends IntDecoder {
@Override
- protected void doDecode(BytesRef buf, IntsRef values, int upto) {
- while (buf.offset < upto) {
- if (values.length == values.ints.length) {
- values.grow(values.length + 10); // grow by few items, however not too many
+ public void decode(BytesRef buf, IntsRef values) {
+ values.offset = values.length = 0;
+
+ // grow the buffer up front, even if by a large number of values (buf.length)
+ // that saves the need to check inside the loop for every decoded value if
+ // the buffer needs to grow.
+ if (values.ints.length < buf.length) {
+ values.ints = new int[ArrayUtil.oversize(buf.length, RamUsageEstimator.NUM_BYTES_INT)];
+ }
+
+ // it is better if the decoding is inlined like so, and not e.g.
+ // in a utility method
+ int upto = buf.offset + buf.length;
+ int value = 0;
+ int offset = buf.offset;
+ while (offset < upto) {
+ byte b = buf.bytes[offset++];
+ if (b >= 0) {
+ values.ints[values.length++] = (value << 7) | b;
+ value = 0;
+ } else {
+ value = (value << 7) | (b & 0x7F);
}
- values.ints[values.length++] = VInt8.decode(buf);
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/java/org/apache/lucene/util/encoding/VInt8IntEncoder.java Wed Jan 16 13:28:58 2013
@@ -47,17 +47,47 @@ import org.apache.lucene.util.IntsRef;
*
* @lucene.experimental
*/
-public class VInt8IntEncoder extends IntEncoder {
+public final class VInt8IntEncoder extends IntEncoder {
@Override
- protected void doEncode(IntsRef values, BytesRef buf, int upto) {
+ public void encode(IntsRef values, BytesRef buf) {
+ buf.offset = buf.length = 0;
int maxBytesNeeded = 5 * values.length; // at most 5 bytes per VInt
if (buf.bytes.length < maxBytesNeeded) {
buf.grow(maxBytesNeeded);
}
+ int upto = values.offset + values.length;
for (int i = values.offset; i < upto; i++) {
- VInt8.encode(values.ints[i], buf);
+ // it is better if the encoding is inlined like so, and not e.g.
+ // in a utility method
+ int value = values.ints[i];
+ if ((value & ~0x7F) == 0) {
+ buf.bytes[buf.length] = (byte) value;
+ buf.length++;
+ } else if ((value & ~0x3FFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 1] = (byte) (value & 0x7F);
+ buf.length += 2;
+ } else if ((value & ~0x1FFFFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 2] = (byte) (value & 0x7F);
+ buf.length += 3;
+ } else if ((value & ~0xFFFFFFF) == 0) {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 3] = (byte) (value & 0x7F);
+ buf.length += 4;
+ } else {
+ buf.bytes[buf.length] = (byte) (0x80 | ((value & 0xF0000000) >> 28));
+ buf.bytes[buf.length + 1] = (byte) (0x80 | ((value & 0xFE00000) >> 21));
+ buf.bytes[buf.length + 2] = (byte) (0x80 | ((value & 0x1FC000) >> 14));
+ buf.bytes[buf.length + 3] = (byte) (0x80 | ((value & 0x3F80) >> 7));
+ buf.bytes[buf.length + 4] = (byte) (value & 0x7F);
+ buf.length += 5;
+ }
}
}
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/facet/index/params/CategoryListParamsTest.java Wed Jan 16 13:28:58 2013
@@ -1,6 +1,12 @@
package org.apache.lucene.facet.index.params;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.encoding.DGapIntEncoder;
+import org.apache.lucene.util.encoding.IntDecoder;
+import org.apache.lucene.util.encoding.IntEncoder;
+import org.apache.lucene.util.encoding.SortingIntEncoder;
+import org.apache.lucene.util.encoding.UniqueValuesIntEncoder;
+import org.apache.lucene.util.encoding.VInt8IntEncoder;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
@@ -29,8 +35,10 @@ public class CategoryListParamsTest exte
public void testDefaultSettings() {
CategoryListParams clp = new CategoryListParams();
assertEquals("wrong default term", new Term("$facets", "$fulltree$"), clp.getTerm());
- assertEquals("unexpected default encoder", "Sorting (Unique (DGap (VInt8)))", clp.createEncoder().toString());
- assertEquals("unexpected default decoder", "DGap (VInt8)", clp.createEncoder().createMatchingDecoder().toString());
+ IntEncoder encoder = new SortingIntEncoder(new UniqueValuesIntEncoder(new DGapIntEncoder(new VInt8IntEncoder())));
+ IntDecoder decoder = encoder.createMatchingDecoder();
+ assertEquals("unexpected default encoder", encoder.toString(), clp.createEncoder().toString());
+ assertEquals("unexpected default decoder", decoder.toString(), clp.createEncoder().createMatchingDecoder().toString());
}
/**
Modified: lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java?rev=1433930&r1=1433929&r2=1433930&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java (original)
+++ lucene/dev/branches/lucene_solr_4_1/lucene/facet/src/test/org/apache/lucene/util/encoding/EncodingTest.java Wed Jan 16 13:28:58 2013
@@ -64,9 +64,12 @@ public class EncodingTest extends Lucene
BytesRef bytes = new BytesRef(100); // some initial capacity - encoders should grow the byte[]
IntsRef values = new IntsRef(100); // some initial capacity - decoders should grow the int[]
- encoding(encoder, data, bytes);
- decoding(bytes, values, encoder.createMatchingDecoder());
- assertTrue(expected.intsEquals(values));
+ for (int i = 0; i < 2; i++) {
+ // run 2 iterations to catch encoders/decoders which don't reset properly
+ encoding(encoder, data, bytes);
+ decoding(bytes, values, encoder.createMatchingDecoder());
+ assertTrue(expected.intsEquals(values));
+ }
}
private static void encoding(IntEncoder encoder, IntsRef data, BytesRef bytes) throws IOException {