You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/11 08:38:40 UTC
kylin git commit: KYLIN-2497 Bug fix and fix UT
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2497-2 7033f4be5 -> 6672678ac
KYLIN-2497 Bug fix and fix UT
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6672678a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6672678a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6672678a
Branch: refs/heads/KYLIN-2497-2
Commit: 6672678ac3ade4f43277a7c6d1c034d894fc105b
Parents: 7033f4b
Author: xiefan46 <95...@qq.com>
Authored: Sat Mar 11 14:53:16 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Sat Mar 11 16:36:58 2017 +0800
----------------------------------------------------------------------
.../engine/mr/steps/SelfDefineSortableKey.java | 4 +-
.../mr/steps/SelfDefineSortableKeyOld.java | 143 -------------------
.../mr/steps/NumberDictionaryForestTest.java | 33 ++---
.../mr/steps/SelfDefineSortableKeyTest.java | 42 +++---
4 files changed, 40 insertions(+), 182 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6672678a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index ace753a..0c541b2 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -50,7 +50,7 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
if (isNumberFamily()) {
String valueStr = new String(key.getBytes(), 1, key.getLength() - 1);
if (isIntegerFamily()) {
- this.keyInObj = Integer.parseInt(valueStr);
+ this.keyInObj = Long.parseLong(valueStr);
} else {
this.keyInObj = Double.parseDouble(valueStr);
}
@@ -73,7 +73,7 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
return ((Text) this.keyInObj).compareTo(((Text) o.keyInObj));
} else {
if (isIntegerFamily()) {
- return Integer.compare((Integer) this.keyInObj, (Integer) o.keyInObj);
+ return Long.compare((Long) this.keyInObj, (Long) o.keyInObj);
} else {
return Double.compare((Double) this.keyInObj, (Double) o.keyInObj);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6672678a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
deleted file mode 100644
index e360b49..0000000
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-package org.apache.kylin.engine.mr.steps;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-/**
- * Created by xiefan on 16-11-1.
- */
-public class SelfDefineSortableKeyOld implements WritableComparable<SelfDefineSortableKeyOld> {
- public enum TypeFlag {
- NONE_NUMERIC_TYPE,
- INTEGER_FAMILY_TYPE,
- DOUBLE_FAMILY_TYPE
- }
-
- private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010)
-
- private Text text;
-
- private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKeyOld.class);
-
- public SelfDefineSortableKeyOld() {
- }
-
- public SelfDefineSortableKeyOld(byte typeId, Text text) {
- this.typeId = typeId;
- this.text = text;
- }
-
- @Override
- public int compareTo(SelfDefineSortableKeyOld o) {
- if (!o.isNumberFamily()) {
- return this.text.compareTo(o.text);
- } else {
- byte[] data1 = this.text.getBytes();
- byte[] data2 = o.text.getBytes();
- String str1 = new String(data1, 1, data1.length - 1);
- String str2 = new String(data2, 1, data2.length - 1);
- if (str1.equals("") || str2.equals("")) {
- //should not achieve here
- logger.error("none numeric value!");
- return 0;
- }
- if (o.isIntegerFamily()) { //integer type
- try {
- Long num1 = Long.parseLong(str1);
- Long num2 = Long.parseLong(str2);
- return num1.compareTo(num2);
- } catch (NumberFormatException e) {
- logger.error("NumberFormatException when parse integer family number.str1:" + str1 + " str2:" + str2);
- e.printStackTrace();
- return 0;
- }
- } else { //other numeric type
- try {
- Double num1 = Double.parseDouble(str1);
- Double num2 = Double.parseDouble(str2);
- return num1.compareTo(num2);
- } catch (NumberFormatException e) {
- logger.error("NumberFormatException when parse doul family number.str1:" + str1 + " str2:" + str2);
- return 0;
- }
- }
- }
- }
-
- @Override
- public void write(DataOutput dataOutput) throws IOException {
- dataOutput.writeByte(typeId);
- text.write(dataOutput);
- }
-
- @Override
- public void readFields(DataInput dataInput) throws IOException {
- this.typeId = dataInput.readByte();
- this.text = new Text();
- text.readFields(dataInput);
- }
-
- public short getTypeId() {
- return typeId;
- }
-
- public Text getText() {
- return text;
- }
-
- public boolean isNumberFamily() {
- if (typeId == TypeFlag.NONE_NUMERIC_TYPE.ordinal())
- return false;
- return true;
- }
-
- public boolean isIntegerFamily() {
- return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
- }
-
- public boolean isOtherNumericFamily() {
- return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
- }
-
- public void setTypeIdByDatatype(DataType type) {
- if (!type.isNumberFamily()) {
- this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
- } else if (type.isIntegerFamily()) {
- this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
- } else {
- this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
- }
- }
-
- public void setTypeId(byte typeId) {
- this.typeId = typeId;
- }
-
- public void setText(Text text) {
- this.text = text;
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6672678a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index f3209cc..c31377c 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -39,7 +39,6 @@ import org.apache.kylin.dict.NumberDictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.TrieDictionaryForest;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag;
import org.junit.Ignore;
import org.junit.Test;
@@ -49,33 +48,34 @@ import org.junit.Test;
public class NumberDictionaryForestTest {
+
@Test
public void testNumberDictionaryForestLong() {
List<String> list = randomLongData(100);
- testData(list, TypeFlag.INTEGER_FAMILY_TYPE);
+ testData(list, SelfDefineSortableKey.TypeFlag.INTEGER_FAMILY_TYPE);
}
@Test
public void testNumberDictionaryForestDouble() {
List<String> list = randomDoubleData(100);
- testData(list, TypeFlag.DOUBLE_FAMILY_TYPE);
+ testData(list, SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE);
}
- private void testData(List<String> list, TypeFlag flag) {
+ private void testData(List<String> list, SelfDefineSortableKey.TypeFlag flag) {
//stimulate map-reduce job
- ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(list, (byte) flag.ordinal());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(list, (byte) flag.ordinal());
Collections.sort(keyList);
//build tree
NumberDictionaryForestBuilder b = new NumberDictionaryForestBuilder(0, 0);
- for (SelfDefineSortableKeyOld key : keyList) {
+ for (SelfDefineSortableKey key : keyList) {
String fieldValue = printKey(key);
b.addValue(fieldValue);
}
TrieDictionaryForest<String> dict = b.build();
dict.dump(System.out);
ArrayList<Integer> resultIds = new ArrayList<>();
- for (SelfDefineSortableKeyOld key : keyList) {
+ for (SelfDefineSortableKey key : keyList) {
String fieldValue = getFieldValue(key);
resultIds.add(dict.getIdFromValue(fieldValue));
assertEquals(fieldValue, dict.getValueFromId(dict.getIdFromValue(fieldValue)));
@@ -127,7 +127,7 @@ public class NumberDictionaryForestTest {
dict2.dump(System.out);
}
-
+
@Test
public void testMerge() {
// mimic the logic as in MergeCuboidMapper
@@ -136,7 +136,7 @@ public class NumberDictionaryForestTest {
b1.addValue("3");
b1.addValue("23");
TrieDictionaryForest<String> dict1 = b1.build();
-
+
NumberDictionaryForestBuilder b2 = new NumberDictionaryForestBuilder();
b2.addValue("0");
b2.addValue("2");
@@ -144,7 +144,7 @@ public class NumberDictionaryForestTest {
b2.addValue("15");
b2.addValue("23");
TrieDictionaryForest<String> dict2 = b2.build();
-
+
assertTrue(dict1.getSizeOfId() == dict2.getSizeOfId());
assertTrue(dict1.getSizeOfValue() == dict2.getSizeOfValue());
@@ -167,7 +167,7 @@ public class NumberDictionaryForestTest {
@Ignore
@Test
- public void testDecimalsWithBeginZero(){
+ public void testDecimalsWithBeginZero() {
List<String> testData = new ArrayList<>();
testData.add("000000000000000000000000000.4868");
testData.add("00000000000000000000000000000000000000");
@@ -221,9 +221,9 @@ public class NumberDictionaryForestTest {
return list;
}
- private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList, byte typeFlag) {
+ private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) {
int partationId = 0;
- ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>();
+ ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>();
for (String str : strNumList) {
ByteBuffer keyBuffer = ByteBuffer.allocate(4096);
int offset = keyBuffer.position();
@@ -235,20 +235,21 @@ public class NumberDictionaryForestTest {
//System.out.println("arrays toString:"+Arrays.toString(valueField));
Text outputKey = new Text();
outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag, outputKey);
+ SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+ sortableKey.init(outputKey, typeFlag);
keyList.add(sortableKey);
}
return keyList;
}
- private String printKey(SelfDefineSortableKeyOld key) {
+ private String printKey(SelfDefineSortableKey key) {
Text data = key.getText();
String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);
return fieldValue;
}
- private String getFieldValue(SelfDefineSortableKeyOld key) {
+ private String getFieldValue(SelfDefineSortableKey key) {
Text data = key.getText();
return Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6672678a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
index dc30029..551998f 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
@@ -18,6 +18,7 @@
package org.apache.kylin.engine.mr.steps;
+import static junit.framework.TestCase.fail;
import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer;
@@ -31,7 +32,6 @@ import java.util.UUID;
import org.apache.hadoop.io.Text;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag;
import org.junit.Test;
/**
@@ -56,11 +56,11 @@ public class SelfDefineSortableKeyTest {
System.out.println("test numbers:" + longList);
ArrayList<String> strNumList = listToStringList(longList);
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
System.out.println(keyList.get(0).isIntegerFamily());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
- for (SelfDefineSortableKeyOld key : keyList) {
+ for (SelfDefineSortableKey key : keyList) {
String str = printKey(key);
strListAftereSort.add(str);
}
@@ -92,11 +92,10 @@ public class SelfDefineSortableKeyTest {
System.out.println("test numbers:" + doubleList);
ArrayList<String> strNumList = listToStringList(doubleList);
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
- System.out.println(keyList.get(0).isOtherNumericFamily());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
- for (SelfDefineSortableKeyOld key : keyList) {
+ for (SelfDefineSortableKey key : keyList) {
String str = printKey(key);
strListAftereSort.add(str);
}
@@ -122,11 +121,10 @@ public class SelfDefineSortableKeyTest {
strList.add("hello"); //duplicate
strList.add("123");
strList.add("");
- ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strList, (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal());
- System.out.println(keyList.get(0).isOtherNumericFamily());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strList, (byte) SelfDefineSortableKey.TypeFlag.NONE_NUMERIC_TYPE.ordinal());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
- for (SelfDefineSortableKeyOld key : keyList) {
+ for (SelfDefineSortableKey key : keyList) {
String str = printKey(key);
strListAftereSort.add(str);
}
@@ -156,7 +154,7 @@ public class SelfDefineSortableKeyTest {
ArrayList<String> strNumList = listToStringList(doubleList);
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
System.out.println("start to test str sort");
long start = System.currentTimeMillis();
@@ -182,7 +180,7 @@ public class SelfDefineSortableKeyTest {
List<SelfDefineSortableKey> newKeyList = new ArrayList<>();
for (String str : strNumList) {
SelfDefineSortableKey key = new SelfDefineSortableKey();
- key.init(new Text(str), (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ key.init(new Text(str), (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
newKeyList.add(key);
}
System.out.println("start to test new sortable key");
@@ -211,18 +209,19 @@ public class SelfDefineSortableKeyTest {
ArrayList<String> strNumList = listToStringList(doubleList);
strNumList.add("fjaeif"); //illegal type
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
- System.out.println(keyList.get(0).isOtherNumericFamily());
- Collections.sort(keyList);
- for (SelfDefineSortableKeyOld key : keyList) {
- printKey(key);
+ try {
+ ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ Collections.sort(keyList);
+ fail("Need catch exception");
+ }catch(Exception e){
+ //correct
}
}
@Test
public void testEnum() {
- TypeFlag flag = TypeFlag.DOUBLE_FAMILY_TYPE;
+ SelfDefineSortableKey.TypeFlag flag = SelfDefineSortableKey.TypeFlag.DOUBLE_FAMILY_TYPE;
System.out.println((byte) flag.ordinal());
int t = (byte) flag.ordinal();
System.out.println(t);
@@ -236,9 +235,9 @@ public class SelfDefineSortableKeyTest {
return strList;
}
- private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList, byte typeFlag) {
+ private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) {
int partationId = 0;
- ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>();
+ ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>();
for (String str : strNumList) {
ByteBuffer keyBuffer = ByteBuffer.allocate(4096);
int offset = keyBuffer.position();
@@ -247,13 +246,14 @@ public class SelfDefineSortableKeyTest {
Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1);
Text outputKey = new Text();
outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag, outputKey);
+ SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+ sortableKey.init(outputKey, typeFlag);
keyList.add(sortableKey);
}
return keyList;
}
- private String printKey(SelfDefineSortableKeyOld key) {
+ private String printKey(SelfDefineSortableKey key) {
Text data = key.getText();
String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);