You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/10 10:41:10 UTC
kylin git commit: KYLIN-2497 Optimize the performance of
SelfDefineSortableKey
Repository: kylin
Updated Branches:
refs/heads/KYLIN-2497 [created] 6e89e7796
KYLIN-2497 Optimize the performance of SelfDefineSortableKey
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e89e779
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e89e779
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e89e779
Branch: refs/heads/KYLIN-2497
Commit: 6e89e7796455e85065730353216ae9b6891b5e0e
Parents: 8edf234
Author: xiefan46 <95...@qq.com>
Authored: Fri Mar 10 17:20:53 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Mar 10 18:38:05 2017 +0800
----------------------------------------------------------------------
.../mr/steps/FactDistinctColumnsMapper.java | 18 ++-
.../mr/steps/FactDistinctColumnsMapperBase.java | 2 +-
.../engine/mr/steps/SelfDefineSortableKey.java | 102 ++++++-------
.../mr/steps/SelfDefineSortableKeyOld.java | 143 +++++++++++++++++++
.../mr/steps/NumberDictionaryForestTest.java | 18 +--
.../mr/steps/SelfDefineSortableKeyTest.java | 82 +++++++++--
6 files changed, 278 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 07e788c..9f65163 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -31,6 +31,7 @@ import org.apache.kylin.cube.cuboid.CuboidScheduler;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.measure.BufferedMeasureCodec;
import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.TblColRef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,6 +41,7 @@ import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
+
/**
*/
public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperBase<KEYIN, Object> {
@@ -50,6 +52,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
BYTES
}
+
protected boolean collectStatistics = false;
protected CuboidScheduler cuboidScheduler = null;
protected int nRowKey;
@@ -68,6 +71,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
private int partitionColumnIndex = -1;
private boolean needFetchPartitionCol = true;
+ private SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+
@Override
protected void setup(Context context) throws IOException {
super.setup(context);
@@ -137,7 +142,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
String[] row = flatTableInputFormat.parseMapperInput(record);
context.getCounter(RawDataCounter.BYTES).increment(countSizeInBytes(row));
-
for (int i = 0; i < factDictCols.size(); i++) {
String fieldValue = row[dictionaryColumnIndex[i]];
if (fieldValue == null)
@@ -161,9 +165,9 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
tmpbuf.put(Bytes.toBytes(reducerIndex)[3]);
tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
- sortableKey.setText(outputKey);
+ DataType type = factDictCols.get(i).getType();
+ sortableKey.init(outputKey, type);
//judge type
- sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType());
context.write(sortableKey, EMPTY_TEXT);
// log a few rows for troubleshooting
@@ -189,8 +193,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
tmpbuf.put(MARK_FOR_PARTITION_COL);
tmpbuf.put(valueBytes);
outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
- sortableKey.setText(outputKey);
- sortableKey.setTypeId((byte) 0);
+ sortableKey.init(outputKey, (byte) 0);
context.write(sortableKey, EMPTY_TEXT);
}
}
@@ -247,13 +250,13 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
hllBuf.clear();
hll.writeRegisters(hllBuf);
outputValue.set(hllBuf.array(), 0, hllBuf.position());
- sortableKey.setText(outputKey);
- sortableKey.setTypeId((byte) 0);
+ sortableKey.init(outputKey, (byte) 0);
context.write(sortableKey, outputValue);
}
}
}
+
private int countNewSize(int oldSize, int dataSize) {
int newSize = oldSize * 2;
while (newSize < dataSize) {
@@ -261,4 +264,5 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
}
return newSize;
}
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 34624e3..458af69 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -54,7 +54,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN> extends Kyli
protected IMRTableInputFormat flatTableInputFormat;
protected Text outputKey = new Text();
- protected SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+ //protected SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
protected Text outputValue = new Text();
protected int errorRecordCounter = 0;
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index 74525d3..48c7596 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -17,20 +17,19 @@
*/
package org.apache.kylin.engine.mr.steps;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.metadata.datatype.DataType;
+
/**
- * Created by xiefan on 16-11-1.
+ * Created by xiefan on 17-3-10.
*/
public class SelfDefineSortableKey implements WritableComparable<SelfDefineSortableKey> {
+
public enum TypeFlag {
NONE_NUMERIC_TYPE,
INTEGER_FAMILY_TYPE,
@@ -39,51 +38,44 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010)
- private Text text;
+ private Object keyInObj;
- private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKey.class);
public SelfDefineSortableKey() {
}
- public SelfDefineSortableKey(byte typeId, Text text) {
+ public SelfDefineSortableKey(Text key, DataType type) {
+ init(key, getTypeIdByDatatype(type));
+ }
+
+ public void init(Text key, byte typeId) {
this.typeId = typeId;
- this.text = text;
+ if (isNumberFamily()) {
+ if (isIntegerFamily()) {
+ this.keyInObj = Integer.parseInt(new String(key.getBytes()));
+ } else {
+ this.keyInObj = Double.parseDouble(new String(key.getBytes()));
+ }
+ } else {
+ this.keyInObj = key;
+ }
+ }
+
+ public void init(Text key, DataType type) {
+ init(key, getTypeIdByDatatype(type));
}
@Override
public int compareTo(SelfDefineSortableKey o) {
- if (!o.isNumberFamily()) {
- return this.text.compareTo(o.text);
+ if (this.typeId != o.typeId)
+ throw new IllegalStateException("Error. Incompatible types");
+ if (!isNumberFamily()) {
+ return ((Text) this.keyInObj).compareTo(((Text) o.keyInObj));
} else {
- byte[] data1 = this.text.getBytes();
- byte[] data2 = o.text.getBytes();
- String str1 = new String(data1, 1, data1.length - 1);
- String str2 = new String(data2, 1, data2.length - 1);
- if (str1.equals("") || str2.equals("")) {
- //should not achieve here
- logger.error("none numeric value!");
- return 0;
- }
- if (o.isIntegerFamily()) { //integer type
- try {
- Long num1 = Long.parseLong(str1);
- Long num2 = Long.parseLong(str2);
- return num1.compareTo(num2);
- } catch (NumberFormatException e) {
- logger.error("NumberFormatException when parse integer family number.str1:" + str1 + " str2:" + str2);
- e.printStackTrace();
- return 0;
- }
- } else { //other numeric type
- try {
- Double num1 = Double.parseDouble(str1);
- Double num2 = Double.parseDouble(str2);
- return num1.compareTo(num2);
- } catch (NumberFormatException e) {
- logger.error("NumberFormatException when parse doul family number.str1:" + str1 + " str2:" + str2);
- return 0;
- }
+ if (isIntegerFamily()) {
+ return Integer.compare((Integer) this.keyInObj, (Integer) o.keyInObj);
+ } else {
+ return Double.compare((Double) this.keyInObj, (Double) o.keyInObj);
}
}
}
@@ -91,14 +83,15 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
@Override
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeByte(typeId);
- text.write(dataOutput);
+ Text outputKey = new Text(keyInObj.toString());
+ outputKey.write(dataOutput);
}
@Override
public void readFields(DataInput dataInput) throws IOException {
this.typeId = dataInput.readByte();
- this.text = new Text();
- text.readFields(dataInput);
+ Text inputKey = new Text();
+ init(inputKey, typeId);
}
public short getTypeId() {
@@ -106,7 +99,8 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
}
public Text getText() {
- return text;
+ Text key = new Text(keyInObj.toString());
+ return key;
}
public boolean isNumberFamily() {
@@ -119,25 +113,21 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
}
- public boolean isOtherNumericFamily() {
- return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
- }
- public void setTypeIdByDatatype(DataType type) {
+ public byte getTypeIdByDatatype(DataType type) {
if (!type.isNumberFamily()) {
- this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
+ return (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
} else if (type.isIntegerFamily()) {
- this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
+ return (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
} else {
- this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
+ return (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
}
}
-
+
public void setTypeId(byte typeId) {
this.typeId = typeId;
}
- public void setText(Text text) {
- this.text = text;
- }
}
+
+
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
new file mode 100644
index 0000000..e360b49
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.engine.mr.steps;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Created by xiefan on 16-11-1.
+ */
+public class SelfDefineSortableKeyOld implements WritableComparable<SelfDefineSortableKeyOld> {
+ public enum TypeFlag {
+ NONE_NUMERIC_TYPE,
+ INTEGER_FAMILY_TYPE,
+ DOUBLE_FAMILY_TYPE
+ }
+
+ private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010)
+
+ private Text text;
+
+ private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKeyOld.class);
+
+ public SelfDefineSortableKeyOld() {
+ }
+
+ public SelfDefineSortableKeyOld(byte typeId, Text text) {
+ this.typeId = typeId;
+ this.text = text;
+ }
+
+ @Override
+ public int compareTo(SelfDefineSortableKeyOld o) {
+ if (!o.isNumberFamily()) {
+ return this.text.compareTo(o.text);
+ } else {
+ byte[] data1 = this.text.getBytes();
+ byte[] data2 = o.text.getBytes();
+ String str1 = new String(data1, 1, data1.length - 1);
+ String str2 = new String(data2, 1, data2.length - 1);
+ if (str1.equals("") || str2.equals("")) {
+ //should not achieve here
+ logger.error("none numeric value!");
+ return 0;
+ }
+ if (o.isIntegerFamily()) { //integer type
+ try {
+ Long num1 = Long.parseLong(str1);
+ Long num2 = Long.parseLong(str2);
+ return num1.compareTo(num2);
+ } catch (NumberFormatException e) {
+ logger.error("NumberFormatException when parse integer family number.str1:" + str1 + " str2:" + str2);
+ e.printStackTrace();
+ return 0;
+ }
+ } else { //other numeric type
+ try {
+ Double num1 = Double.parseDouble(str1);
+ Double num2 = Double.parseDouble(str2);
+ return num1.compareTo(num2);
+ } catch (NumberFormatException e) {
+ logger.error("NumberFormatException when parse doul family number.str1:" + str1 + " str2:" + str2);
+ return 0;
+ }
+ }
+ }
+ }
+
+ @Override
+ public void write(DataOutput dataOutput) throws IOException {
+ dataOutput.writeByte(typeId);
+ text.write(dataOutput);
+ }
+
+ @Override
+ public void readFields(DataInput dataInput) throws IOException {
+ this.typeId = dataInput.readByte();
+ this.text = new Text();
+ text.readFields(dataInput);
+ }
+
+ public short getTypeId() {
+ return typeId;
+ }
+
+ public Text getText() {
+ return text;
+ }
+
+ public boolean isNumberFamily() {
+ if (typeId == TypeFlag.NONE_NUMERIC_TYPE.ordinal())
+ return false;
+ return true;
+ }
+
+ public boolean isIntegerFamily() {
+ return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
+ }
+
+ public boolean isOtherNumericFamily() {
+ return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ }
+
+ public void setTypeIdByDatatype(DataType type) {
+ if (!type.isNumberFamily()) {
+ this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
+ } else if (type.isIntegerFamily()) {
+ this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
+ } else {
+ this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
+ }
+ }
+
+ public void setTypeId(byte typeId) {
+ this.typeId = typeId;
+ }
+
+ public void setText(Text text) {
+ this.text = text;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index 1908992..f3209cc 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -39,7 +39,7 @@ import org.apache.kylin.dict.NumberDictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.TrieDictionaryForest;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
+import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag;
import org.junit.Ignore;
import org.junit.Test;
@@ -63,19 +63,19 @@ public class NumberDictionaryForestTest {
private void testData(List<String> list, TypeFlag flag) {
//stimulate map-reduce job
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(list, (byte) flag.ordinal());
+ ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(list, (byte) flag.ordinal());
Collections.sort(keyList);
//build tree
NumberDictionaryForestBuilder b = new NumberDictionaryForestBuilder(0, 0);
- for (SelfDefineSortableKey key : keyList) {
+ for (SelfDefineSortableKeyOld key : keyList) {
String fieldValue = printKey(key);
b.addValue(fieldValue);
}
TrieDictionaryForest<String> dict = b.build();
dict.dump(System.out);
ArrayList<Integer> resultIds = new ArrayList<>();
- for (SelfDefineSortableKey key : keyList) {
+ for (SelfDefineSortableKeyOld key : keyList) {
String fieldValue = getFieldValue(key);
resultIds.add(dict.getIdFromValue(fieldValue));
assertEquals(fieldValue, dict.getValueFromId(dict.getIdFromValue(fieldValue)));
@@ -221,9 +221,9 @@ public class NumberDictionaryForestTest {
return list;
}
- private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) {
+ private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList, byte typeFlag) {
int partationId = 0;
- ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>();
+ ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>();
for (String str : strNumList) {
ByteBuffer keyBuffer = ByteBuffer.allocate(4096);
int offset = keyBuffer.position();
@@ -235,20 +235,20 @@ public class NumberDictionaryForestTest {
//System.out.println("arrays toString:"+Arrays.toString(valueField));
Text outputKey = new Text();
outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey);
+ SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag, outputKey);
keyList.add(sortableKey);
}
return keyList;
}
- private String printKey(SelfDefineSortableKey key) {
+ private String printKey(SelfDefineSortableKeyOld key) {
Text data = key.getText();
String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);
return fieldValue;
}
- private String getFieldValue(SelfDefineSortableKey key) {
+ private String getFieldValue(SelfDefineSortableKeyOld key) {
Text data = key.getText();
return Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
index df6287f..dc30029 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
@@ -29,8 +29,9 @@ import java.util.Random;
import java.util.UUID;
import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
+import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag;
import org.junit.Test;
/**
@@ -55,11 +56,11 @@ public class SelfDefineSortableKeyTest {
System.out.println("test numbers:" + longList);
ArrayList<String> strNumList = listToStringList(longList);
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
+ ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
System.out.println(keyList.get(0).isIntegerFamily());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
- for (SelfDefineSortableKey key : keyList) {
+ for (SelfDefineSortableKeyOld key : keyList) {
String str = printKey(key);
strListAftereSort.add(str);
}
@@ -91,11 +92,11 @@ public class SelfDefineSortableKeyTest {
System.out.println("test numbers:" + doubleList);
ArrayList<String> strNumList = listToStringList(doubleList);
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
System.out.println(keyList.get(0).isOtherNumericFamily());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
- for (SelfDefineSortableKey key : keyList) {
+ for (SelfDefineSortableKeyOld key : keyList) {
String str = printKey(key);
strListAftereSort.add(str);
}
@@ -121,11 +122,11 @@ public class SelfDefineSortableKeyTest {
strList.add("hello"); //duplicate
strList.add("123");
strList.add("");
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strList, (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal());
+ ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strList, (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal());
System.out.println(keyList.get(0).isOtherNumericFamily());
Collections.sort(keyList);
ArrayList<String> strListAftereSort = new ArrayList<>();
- for (SelfDefineSortableKey key : keyList) {
+ for (SelfDefineSortableKeyOld key : keyList) {
String str = printKey(key);
strListAftereSort.add(str);
}
@@ -138,6 +139,60 @@ public class SelfDefineSortableKeyTest {
}
@Test
+ public void testPerformance() {
+ Random rand = new Random(System.currentTimeMillis());
+ ArrayList<Double> doubleList = new ArrayList<>();
+ int count = 10 * 10000;
+ for (int i = 0; i < count; i++) {
+ doubleList.add(rand.nextDouble());
+ }
+ doubleList.add(0.0);
+ doubleList.add(0.0); //test duplicate
+ doubleList.add(-1.0); //test negative number
+ doubleList.add(Double.MAX_VALUE);
+ doubleList.add(-Double.MAX_VALUE);
+ //System.out.println(Double.MIN_VALUE);
+
+
+ ArrayList<String> strNumList = listToStringList(doubleList);
+ //System.out.println("test num strs list:"+strNumList);
+ ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+
+ System.out.println("start to test str sort");
+ long start = System.currentTimeMillis();
+ Collections.sort(strNumList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+
+ System.out.println("start to test double sort");
+ start = System.currentTimeMillis();
+ Collections.sort(keyList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+ List<ByteArray> byteList = new ArrayList<>();
+ for (String str : strNumList) {
+ byteList.add(new ByteArray(Bytes.toBytes(str)));
+ }
+ System.out.println("start to test byte array sort");
+ start = System.currentTimeMillis();
+ Collections.sort(byteList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+ //test new key
+ List<SelfDefineSortableKey> newKeyList = new ArrayList<>();
+ for (String str : strNumList) {
+ SelfDefineSortableKey key = new SelfDefineSortableKey();
+ key.init(new Text(str), (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ newKeyList.add(key);
+ }
+ System.out.println("start to test new sortable key");
+ start = System.currentTimeMillis();
+ Collections.sort(newKeyList);
+ System.out.println("sort time : " + (System.currentTimeMillis() - start));
+ }
+
+
+ @Test
public void testIllegalNumber() {
Random rand = new Random(System.currentTimeMillis());
ArrayList<Double> doubleList = new ArrayList<>();
@@ -156,10 +211,10 @@ public class SelfDefineSortableKeyTest {
ArrayList<String> strNumList = listToStringList(doubleList);
strNumList.add("fjaeif"); //illegal type
//System.out.println("test num strs list:"+strNumList);
- ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+ ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
System.out.println(keyList.get(0).isOtherNumericFamily());
Collections.sort(keyList);
- for (SelfDefineSortableKey key : keyList) {
+ for (SelfDefineSortableKeyOld key : keyList) {
printKey(key);
}
@@ -176,15 +231,14 @@ public class SelfDefineSortableKeyTest {
private <T> ArrayList<String> listToStringList(ArrayList<T> list) {
ArrayList<String> strList = new ArrayList<>();
for (T t : list) {
- System.out.println(t.toString());
strList.add(t.toString());
}
return strList;
}
- private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) {
+ private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList, byte typeFlag) {
int partationId = 0;
- ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>();
+ ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>();
for (String str : strNumList) {
ByteBuffer keyBuffer = ByteBuffer.allocate(4096);
int offset = keyBuffer.position();
@@ -193,13 +247,13 @@ public class SelfDefineSortableKeyTest {
Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1);
Text outputKey = new Text();
outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
- SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey);
+ SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag, outputKey);
keyList.add(sortableKey);
}
return keyList;
}
- private String printKey(SelfDefineSortableKey key) {
+ private String printKey(SelfDefineSortableKeyOld key) {
Text data = key.getText();
String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);