You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by wa...@apache.org on 2016/01/12 11:54:47 UTC
[1/2] kylin git commit: KYLIN-1122 add UT and bug fixed
Repository: kylin
Updated Branches:
refs/heads/KYLIN-1122 1e44c595b -> 8814a432d
KYLIN-1122 add UT and bug fixed
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fd1898e5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fd1898e5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fd1898e5
Branch: refs/heads/KYLIN-1122
Commit: fd1898e58c918b0f8bd2698407a910ae8504b3bf
Parents: ab7a76d
Author: Xiaoyu Wang <wa...@apache.org>
Authored: Tue Jan 12 18:53:30 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Tue Jan 12 18:53:30 2016 +0800
----------------------------------------------------------------------
.../kylin/measure/basic/BasicMeasureType.java | 4 --
.../kylin/measure/raw/RawMeasureType.java | 2 +-
.../apache/kylin/measure/raw/RawSerializer.java | 29 +++++----
.../kylin/metadata/model/FunctionDesc.java | 1 -
.../kylin/measure/raw/RawAggregatorTest.java | 35 +++++++++++
.../kylin/measure/raw/RawSerializerTest.java | 64 ++++++++++++++++++++
6 files changed, 118 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
index 9447ab0..b8d201e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BasicMeasureType.java
@@ -77,10 +77,6 @@ public class BasicMeasureType extends MeasureType {
if (rtype.isNumberFamily() == false) {
throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY);
}
- } else if (funcName.equals(FunctionDesc.FUNC_RAW)) {
- if (!FunctionDesc.FUNC_RAW.equalsIgnoreCase(rtype.getName())) {
- throw new IllegalArgumentException("Return type for function " + funcName + " must be raw ");
- }
} else {
KylinConfig config = KylinConfig.getInstanceFromEnv();
if (config.isQueryIgnoreUnknownFunction() == false)
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 92044de..755d7f6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -214,7 +214,7 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
ByteArray raw = rawIterator.next();
int key = BytesUtil.readUnsigned(raw.array(), 0, raw.array().length);
String colValue = rawColDict.getValueFromId(key);
- tuple.setMeasureValue(literalTupleIdx, colValue);
+ tuple.setDimensionValue(literalTupleIdx, colValue);
}
};
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
index 884f777..25c70bc 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawSerializer.java
@@ -30,6 +30,9 @@ import java.util.List;
@SuppressWarnings("unused")
public class RawSerializer extends DataTypeSerializer<List<ByteArray>> {
+ //one dictionary id value need 2 bytes, this buffer can contain ~ 512 * 1024 values
+ //FIXME to config this and RowConstants.ROWVALUE_BUFFER_SIZE in properties file
+ public static final int RAW_BUFFER_SIZE = 1024 * 1024;//1M
public RawSerializer(DataType dataType) {
}
@@ -38,9 +41,9 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> {
public int peekLength(ByteBuffer in) {
int mark = in.position();
int len = 0;
- if(in.hasRemaining()) {
- int size = in.getInt();
- int bytes = in.getInt();
+ if (in.hasRemaining()) {
+ int size = BytesUtil.readVInt(in);
+ int bytes = BytesUtil.readVInt(in);
len = in.position() - mark + bytes;
}
in.position(mark);
@@ -49,26 +52,30 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> {
@Override
public int maxLength() {
- return 1024 * 1024;
+ return RAW_BUFFER_SIZE;
}
@Override
public int getStorageBytesEstimate() {
- return 1024 * 1024;
+ return RAW_BUFFER_SIZE;
}
@Override
public void serialize(List<ByteArray> value, ByteBuffer out) {
- if(value != null) {
+ if (value != null) {
int bytes = 0;
for (ByteArray array : value) {
+ //in BytesUtil.writeByteArray, writeVInt only need 1 byte,
+ //because dictionary id length is less than 127
+ //so one value will need 2 bytes
bytes += (array.length() + 1);
}
+ //size and bytes value will need <=8 bytes
+ BytesUtil.writeVInt(value.size(), out);
+ BytesUtil.writeVInt(bytes, out);
if (bytes > out.remaining()) {
throw new RuntimeException("BufferOverflow! Please use one higher cardinality column for dimension column when build RAW cube!");
}
- out.putInt(value.size());
- out.putInt(bytes);
for (ByteArray array : value) {
BytesUtil.writeByteArray(array.array(), out);
}
@@ -78,9 +85,9 @@ public class RawSerializer extends DataTypeSerializer<List<ByteArray>> {
@Override
public List<ByteArray> deserialize(ByteBuffer in) {
List<ByteArray> value = null;
- if(in.hasRemaining()) {
- int size = in.getInt();
- int bytes = in.getInt();
+ if (in.hasRemaining()) {
+ int size = BytesUtil.readVInt(in);
+ int bytes = BytesUtil.readVInt(in);
value = new ArrayList<ByteArray>(size);
for (int i = 0; i < size; i++) {
value.add(new ByteArray(BytesUtil.readByteArray(in)));
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 624a73c..36c8722 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -40,7 +40,6 @@ public class FunctionDesc {
public static final String FUNC_MIN = "MIN";
public static final String FUNC_MAX = "MAX";
public static final String FUNC_COUNT = "COUNT";
- public static final String FUNC_RAW = "RAW";
public static final String PARAMETER_TYPE_CONSTANT = "constant";
public static final String PARAMETER_TYPE_COLUMN = "column";
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java
new file mode 100644
index 0000000..da54664
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawAggregatorTest.java
@@ -0,0 +1,35 @@
+package org.apache.kylin.measure.raw;
+
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class RawAggregatorTest {
+ private RawAggregator agg = new RawAggregator();
+
+ @Test
+ public void testNormal(){
+ int size = 100;
+ List<ByteArray> valueList = new ArrayList<ByteArray>(size);
+ for (Integer i = 0; i < size; i++) {
+ ByteArray key = new ByteArray(1);
+ BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
+ valueList.add(key);
+ }
+ agg.aggregate(valueList);
+ agg.aggregate(valueList);
+ assertEquals(valueList.size() * 2, agg.getState().size());
+ }
+
+ @Test
+ public void testNull(){
+ agg.aggregate(null);
+ assertEquals(agg.getState().size(), 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/fd1898e5/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java
new file mode 100644
index 0000000..a870559
--- /dev/null
+++ b/core-metadata/src/test/java/org/apache/kylin/measure/raw/RawSerializerTest.java
@@ -0,0 +1,64 @@
+package org.apache.kylin.measure.raw;
+
+import org.apache.kylin.common.util.ByteArray;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class RawSerializerTest {
+ static {
+ DataType.register("raw");
+ }
+
+ private RawSerializer rawSerializer = new RawSerializer(DataType.getType("raw"));
+
+ @Test
+ public void testNormal() {
+ List<ByteArray> input = getValueList(1024);
+ List<ByteArray> output = doSAndD(input);
+ assertEquals(input, output);
+ }
+
+ @Test
+ public void testNull() {
+ List<ByteArray> output = doSAndD(null);
+ assertEquals(output.size(), 0);
+ List<ByteArray> input = new ArrayList<ByteArray>();
+ output = doSAndD(input);
+ assertEquals(input, output);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testOverflow() {
+ List<ByteArray> input = getValueList(512 * 1024);
+ doSAndD(input);
+ }
+
+ private List<ByteArray> doSAndD(List<ByteArray> input) {
+ ByteBuffer out = ByteBuffer.allocate(rawSerializer.maxLength());
+ out.mark();
+ rawSerializer.serialize(input, out);
+ out.reset();
+ return rawSerializer.deserialize(out);
+ }
+
+ private List<ByteArray> getValueList(int size) {
+ if (size == -1) {
+ return null;
+ }
+ List<ByteArray> valueList = new ArrayList<ByteArray>(size);
+ for (Integer i = 0; i < size; i++) {
+ ByteArray key = new ByteArray(1);
+ BytesUtil.writeUnsigned(i, key.array(), 0, key.length());
+ valueList.add(key);
+ }
+ return valueList;
+ }
+
+}
[2/2] kylin git commit: Merge remote-tracking branch
'origin/KYLIN-1122' into KYLIN-1122
Posted by wa...@apache.org.
Merge remote-tracking branch 'origin/KYLIN-1122' into KYLIN-1122
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8814a432
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8814a432
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8814a432
Branch: refs/heads/KYLIN-1122
Commit: 8814a432dacc95c7df3867b06bb6fa97d84fd667
Parents: fd1898e 1e44c59
Author: Xiaoyu Wang <wa...@apache.org>
Authored: Tue Jan 12 18:54:18 2016 +0800
Committer: Xiaoyu Wang <wa...@apache.org>
Committed: Tue Jan 12 18:54:18 2016 +0800
----------------------------------------------------------------------
----------------------------------------------------------------------