You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2012/08/08 17:46:32 UTC
svn commit: r1370817 [3/3] - in
/lucene/dev/branches/pforcodec_3892/lucene/core/src:
java/org/apache/lucene/util/packed/ test/org/apache/lucene/util/packed/
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1370817&r1=1370816&r2=1370817&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Wed Aug 8 15:46:28 2012
@@ -282,6 +282,32 @@ public class PackedInts {
*/
void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations);
+ /**
+ * Read <code>iterations * blockCount()</code> blocks from <code>blocks</code>,
+ * decode them and write <code>iterations * valueCount()</code> values into
+ * <code>values</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start reading blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start writing values
+ * @param iterations controls how much data to decode
+ */
+ void decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations);
+
+ /**
+ * Read <code>8 * iterations * blockCount()</code> blocks from <code>blocks</code>,
+ * decode them and write <code>iterations * valueCount()</code> values into
+ * <code>values</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start reading blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start writing values
+ * @param iterations controls how much data to decode
+ */
+ void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations);
+
}
/**
@@ -326,6 +352,32 @@ public class PackedInts {
*/
void encode(long[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations);
+ /**
+ * Read <code>iterations * valueCount()</code> values from <code>values</code>,
+ * encode them and write <code>iterations * blockCount()</code> blocks into
+ * <code>blocks</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start writing blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start reading values
+ * @param iterations controls how much data to encode
+ */
+ void encode(int[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations);
+
+ /**
+ * Read <code>iterations * valueCount()</code> values from <code>values</code>,
+ * encode them and write <code>8 * iterations * blockCount()</code> blocks into
+ * <code>blocks</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start writing blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start reading values
+ * @param iterations controls how much data to encode
+ */
+ void encode(int[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations);
+
}
/**
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py?rev=1370817&r1=1370816&r2=1370817&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py Wed Aug 8 15:46:28 2012
@@ -53,16 +53,27 @@ enum BulkOperation implements PackedInts
FOOTER="""
- private static long[] toLongArray(byte[] bytes, int offset, int length) {
- final int longsLen = length >>> 3;
- LongBuffer longs = LongBuffer.allocate(longsLen);
- longs.put(ByteBuffer.wrap(bytes, offset, length).asLongBuffer());
- return longs.array();
+ private static long[] toLongArray(int[] ints, int offset, int length) {
+ long[] arr = new long[length];
+ for (int i = 0; i < length; ++i) {
+ arr[i] = ints[offset + i];
+ }
+ return arr;
+ }
+
+ @Override
+ public void decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
+ throw new UnsupportedOperationException();
}
@Override
- public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- decode(toLongArray(blocks, blocksOffset, 8 * iterations * blockCount()), 0, values, valuesOffset, iterations);
+ public void encode(int[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {
+ encode(toLongArray(values, valuesOffset, iterations * valueCount()), 0, blocks, blocksOffset, iterations);
}
@Override
@@ -72,6 +83,13 @@ FOOTER="""
ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer().put(longBLocks);
}
+ @Override
+ public void encode(int[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations) {
+ final long[] longBLocks = new long[blockCount() * iterations];
+ encode(values, valuesOffset, longBLocks, 0, iterations);
+ ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer().put(longBLocks);
+ }
+
/**
* For every number of bits per value, there is a minimum number of
* blocks (b) / values (v) you need to write in order to reach the next block
@@ -139,9 +157,9 @@ def packed64singleblock(bpv, f):
f.write(" public int valueCount() {\n")
f.write(" return %d;\n" %values)
f.write(" }\n\n")
- #p64sb_decode(bpv, 32)
+ p64sb_decode(bpv, 32)
p64sb_decode(bpv, 64)
- #p64sb_encode(bpv, 32)
+ p64sb_encode(bpv, 32)
p64sb_encode(bpv, 64)
f.write(" }")
@@ -178,7 +196,7 @@ def p64sb_decode(bpv, bits):
f.write(" assert blocksOffset + 8 * iterations * blockCount() <= blocks.length;\n")
f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
f.write(" for (int i = 0; i < iterations; ++i) {\n")
- if bpv >= 32:
+ if bpv >= 32 and bits > 32:
for i in xrange(7, -1, -1):
f.write(" final long byte%d = blocks[blocksOffset++] & 0xFF;\n" %i)
else:
@@ -222,6 +240,10 @@ def p64sb_encode(bpv, bits):
typ = get_type(bits)
mask_start, mask_end = masks(bits)
f.write(" public void encode(%s[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {\n" %typ)
+ if bits < bpv:
+ f.write(" throw new UnsupportedOperationException();\n")
+ f.write(" }\n\n")
+ return
f.write(" assert blocksOffset + iterations * blockCount() <= blocks.length;\n")
f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
f.write(" for (int i = 0; i < iterations; ++i) {\n")
@@ -258,6 +280,10 @@ def packed64(bpv, f):
System.arraycopy(blocks, blocksOffset, values, valuesOffset, valueCount() * iterations);
}
+ public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
+ LongBuffer.wrap(values, valuesOffset, iterations * valueCount()).put(ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer());
+ }
+
public void encode(long[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {
System.arraycopy(values, valuesOffset, blocks, blocksOffset, valueCount() * iterations);
}
@@ -265,9 +291,9 @@ def packed64(bpv, f):
}
""")
else:
- #p64_decode(bpv, 32, values)
+ p64_decode(bpv, 32, values)
p64_decode(bpv, 64, values)
- #p64_encode(bpv, 32, values)
+ p64_encode(bpv, 32, values)
p64_encode(bpv, 64, values)
f.write(" }\n")
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1370817&r1=1370816&r2=1370817&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Wed Aug 8 15:46:28 2012
@@ -667,12 +667,27 @@ public class TestPackedInts extends Luce
for (long value : values) {
assertTrue(value <= PackedInts.maxValue(bpv));
}
+ // test decoding to int[]
+ final int[] intValues;
+ if (bpv <= 32) {
+ intValues = new int[values.length];
+ decoder.decode(blocks, blocksOffset, intValues, valuesOffset, iterations);
+ assertTrue(equals(intValues, values));
+ } else {
+ intValues = null;
+ }
// 3. re-encode
final long[] blocks2 = new long[blocksOffset2 + blocksLen];
encoder.encode(values, valuesOffset, blocks2, blocksOffset2, iterations);
assertArrayEquals(msg, Arrays.copyOfRange(blocks, blocksOffset, blocks.length),
Arrays.copyOfRange(blocks2, blocksOffset2, blocks2.length));
+ // test encoding from int[]
+ if (bpv <= 32) {
+ final long[] blocks3 = new long[blocks2.length];
+ encoder.encode(intValues, valuesOffset, blocks3, blocksOffset2, iterations);
+ assertArrayEquals(msg, blocks2, blocks3);
+ }
// 4. byte[] decoding
final byte[] byteBlocks = new byte[8 * blocks.length];
@@ -683,13 +698,37 @@ public class TestPackedInts extends Luce
assertTrue(msg, value <= PackedInts.maxValue(bpv));
}
assertArrayEquals(msg, values, values2);
+ // test decoding to int[]
+ if (bpv <= 32) {
+ final int[] intValues2 = new int[values2.length];
+ decoder.decode(byteBlocks, blocksOffset * 8, intValues2, valuesOffset, iterations);
+ assertTrue(msg, equals(intValues2, values2));
+ }
// 5. byte[] encoding
final byte[] blocks3 = new byte[8 * (blocksOffset2 + blocksLen)];
encoder.encode(values, valuesOffset, blocks3, 8 * blocksOffset2, iterations);
assertEquals(msg, LongBuffer.wrap(blocks2), ByteBuffer.wrap(blocks3).asLongBuffer());
+ // test encoding from int[]
+ if (bpv <= 32) {
+ final byte[] blocks4 = new byte[blocks3.length];
+ encoder.encode(intValues, valuesOffset, blocks4, 8 * blocksOffset2, iterations);
+ assertArrayEquals(msg, blocks3, blocks4);
+ }
+ }
+ }
+ }
+
+ private static boolean equals(int[] ints, long[] longs) {
+ if (ints.length != longs.length) {
+ return false;
+ }
+ for (int i = 0; i < ints.length; ++i) {
+ if ((ints[i] & 0xFFFFFFFFL) != longs[i]) {
+ return false;
}
}
+ return true;
}
}