You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/03/28 01:04:44 UTC
[07/50] incubator-kylin git commit: Bug fix in BytesUtil.writeUnsigned
Bug fix in BytesUtil.writeUnsigned
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/d564876c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/d564876c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/d564876c
Branch: refs/heads/streaming-localdict
Commit: d564876c6776c91895f150e0b9512a18c1e34d35
Parents: 1153150
Author: Shao Feng, Shi <sh...@ebay.com>
Authored: Thu Mar 26 18:03:33 2015 +0800
Committer: Shao Feng, Shi <sh...@ebay.com>
Committed: Thu Mar 26 18:03:33 2015 +0800
----------------------------------------------------------------------
.../common/hll/HyperLogLogPlusCounter.java | 29 ++++++++++++++++++--
.../org/apache/kylin/common/util/BytesUtil.java | 16 +++++------
.../apache/kylin/common/util/BytesUtilTest.java | 20 ++++++++++++++
3 files changed, 55 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d564876c/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
index 686321b..d817bd2 100644
--- a/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
+++ b/common/src/main/java/org/apache/kylin/common/hll/HyperLogLogPlusCounter.java
@@ -186,7 +186,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
BytesUtil.writeVInt(size, out);
for (int i = 0; i < m; i++) {
if (registers[i] > 0) {
- BytesUtil.writeUnsigned(i, indexLen, out);
+ writeUnsigned(i, indexLen, out);
out.put(registers[i]);
}
}
@@ -207,7 +207,7 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
throw new IllegalArgumentException("register size (" + size + ") cannot be larger than m (" + m + ")");
int indexLen = getRegisterIndexSize();
for (int i = 0; i < size; i++) {
- int key = BytesUtil.readUnsigned(in, indexLen);
+ int key = readUnsigned(in, indexLen);
registers[key] = in.get();
}
} else { // array scheme
@@ -306,4 +306,29 @@ public class HyperLogLogPlusCounter implements Comparable<HyperLogLogPlusCounter
System.out.println("HLLC" + p + ",\t" + size + " bytes,\t68% err<" + er + "%" + ",\t95% err<" + er2 + "%" + ",\t99.7% err<" + er3 + "%");
}
}
+
+ /**
+ *
+ * @param num
+ * @param size
+ * @param out
+ */
+ public static void writeUnsigned(int num, int size, ByteBuffer out) {
+ for (int i = 0; i < size; i++) {
+ out.put((byte) num);
+ num >>>= 8;
+ }
+ }
+
+ public static int readUnsigned(ByteBuffer in, int size) {
+ int integer = 0;
+ int mask = 0xff;
+ int shift = 0;
+ for (int i = 0; i < size; i++) {
+ integer |= (in.get() << shift) & mask;
+ mask = mask << 8;
+ shift += 8;
+ }
+ return integer;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d564876c/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
index dbd459d..ca1deaf 100644
--- a/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
+++ b/common/src/main/java/org/apache/kylin/common/util/BytesUtil.java
@@ -184,21 +184,21 @@ public class BytesUtil {
}
public static void writeUnsigned(int num, int size, ByteBuffer out) {
- for (int i = 0; i < size; i++) {
- out.put((byte) num);
- num >>>= 8;
+ int mask = 0xff << ((size - 1) * 8);
+ for (int i = size; i > 0; i--) {
+ int v = (num & mask) >> (i - 1) * 8;
+ out.put((byte) v);
+ mask = mask >> 8;
}
}
public static int readUnsigned(ByteBuffer in, int size) {
int integer = 0;
- int mask = 0xff;
- int shift = 0;
for (int i = 0; i < size; i++) {
- integer |= (in.get() << shift) & mask;
- mask = mask << 8;
- shift += 8;
+ integer = integer << 8;
+ integer += in.get();
}
+
return integer;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/d564876c/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
index 4c88cbe..e34f391 100644
--- a/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
+++ b/common/src/test/java/org/apache/kylin/common/util/BytesUtilTest.java
@@ -22,6 +22,7 @@ import junit.framework.TestCase;
import org.junit.Test;
import java.nio.ByteBuffer;
+import java.util.Arrays;
/**
* by honma
@@ -53,4 +54,23 @@ public class BytesUtilTest extends TestCase {
assertEquals(y[1], false);
}
+ @Test
+ public void testWriteReadUnsignedInt() {
+
+ int testInt = 735033;
+ ByteArray ba = new ByteArray(new byte[3]);
+ BytesUtil.writeUnsigned(testInt, 3, ba.asBuffer());
+
+ byte[] newBytes = new byte[3];
+ System.arraycopy(ba.array(), 0, newBytes, 0, 3);
+ int value = BytesUtil.readUnsigned(new ByteArray(newBytes).asBuffer(), 3);
+
+ assertEquals(value, testInt);
+
+ byte[] anOtherNewBytes = new byte[3];
+ BytesUtil.writeUnsigned(testInt, anOtherNewBytes, 0, 3);
+
+ assertTrue(Arrays.equals(anOtherNewBytes, ba.array()));
+ }
+
}