You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/17 03:32:22 UTC
[17/21] kylin git commit: KYLIN-2400 remove byte[] interface in
dictionary
KYLIN-2400 remove byte[] interface in dictionary
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e769bfa0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e769bfa0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e769bfa0
Branch: refs/heads/master-cdh5.7
Commit: e769bfa004e9d548c5a042f02765cc093ce70ec1
Parents: e20e2b2
Author: xiefan46 <95...@qq.com>
Authored: Mon Jan 9 19:11:39 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Jan 17 10:54:22 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/Dictionary.java | 46 ++++----------------
.../apache/kylin/dict/AppendTrieDictionary.java | 19 ++++----
.../org/apache/kylin/dict/CacheDictionary.java | 13 ++++--
.../apache/kylin/dict/DateStrDictionary.java | 33 --------------
.../apache/kylin/dict/DictionaryGenerator.java | 2 +-
.../org/apache/kylin/dict/NumberDictionary.java | 30 +------------
.../kylin/dict/NumberDictionaryBuilder.java | 8 ----
.../apache/kylin/dict/TimeStrDictionary.java | 29 ------------
.../org/apache/kylin/dict/TrieDictionary.java | 7 ++-
.../kylin/dict/TrieDictionaryBuilder.java | 2 +-
.../apache/kylin/dict/TrieDictionaryForest.java | 27 +++---------
.../kylin/dict/TrieDictionaryForestBuilder.java | 2 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 10 ++---
.../kylin/dict/DateStrDictionaryTest.java | 3 --
.../MultipleDictionaryValueEnumeratorTest.java | 14 ------
.../apache/kylin/dict/NumberDictionaryTest.java | 4 +-
.../kylin/dict/TimeStrDictionaryTest.java | 9 +++-
.../dict/TrieDictionaryForestBenchmark.java | 14 ++----
.../kylin/dict/TrieDictionaryForestTest.java | 11 ++---
.../apache/kylin/dict/TrieDictionaryTest.java | 19 ++++----
.../kylin/measure/bitmap/BitmapMeasureType.java | 11 +++--
.../kylin/measure/raw/RawMeasureType.java | 7 ++-
.../storage/gtrecord/DictGridTableTest.java | 3 +-
.../engine/mr/steps/MergeCuboidMapper.java | 11 ++---
.../mr/steps/NumberDictionaryForestTest.java | 17 +++-----
25 files changed, 92 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 1e172bc..9d675f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -94,8 +94,13 @@ abstract public class Dictionary<T> implements Serializable {
final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException {
if (isNullObjectForm(value))
return nullId();
- else
- return getIdFromValueImpl(value, roundingFlag);
+ else {
+ int id = getIdFromValueImpl(value, roundingFlag);
+ if(id == -1){
+ throw new IllegalArgumentException("Value : " + value + "no exists");
+ }
+ return id;
+ }
}
final public boolean containsValue(T value) throws IllegalArgumentException {
@@ -136,9 +141,7 @@ abstract public class Dictionary<T> implements Serializable {
* Convenient form of
* <code>getIdFromValueBytes(value, offset, len, 0)</code>
*/
- final public int getIdFromValueBytes(byte[] value, int offset, int len) throws IllegalArgumentException {
- return getIdFromValueBytes(value, offset, len, 0);
- }
+
/**
* A lower level API, return ID integer from raw value bytes. In case of not found
@@ -153,31 +156,6 @@ abstract public class Dictionary<T> implements Serializable {
* if value is not found in dictionary and rounding is off;
* or if rounding cannot find a smaller or bigger ID
*/
- final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
- if (isNullByteForm(value, offset, len))
- return nullId();
- else {
- int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- if (id == -1)
- throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
- return id;
- }
- }
-
- protected boolean isNullByteForm(byte[] value, int offset, int len) {
- return value == null;
- }
-
- abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
-
- final public byte[] getValueBytesFromId(int id) {
- if (isNullId(id))
- return BytesUtil.EMPTY_BYTE_ARRAY;
- else
- return getValueBytesFromIdImpl(id);
- }
-
- abstract protected byte[] getValueBytesFromIdImpl(int id);
/**
* A lower level API, get byte values from ID, return the number of bytes
@@ -189,14 +167,6 @@ abstract public class Dictionary<T> implements Serializable {
* @throws IllegalArgumentException
* if ID is not found in dictionary
*/
- final public int getValueBytesFromId(int id, byte[] returnValue, int offset) throws IllegalArgumentException {
- if (isNullId(id))
- return -1;
- else
- return getValueBytesFromIdImpl(id, returnValue, offset);
- }
-
- abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset);
abstract public void dump(PrintStream out);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index b797167..3aef967 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -97,6 +97,8 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
enableCache();
}
+
+
public void initParams(String baseDir, int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter) throws IOException {
this.baseDir = baseDir;
this.baseId = baseId;
@@ -1113,7 +1115,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) {
if (dictSliceMap.isEmpty()) {
return -1;
}
@@ -1129,6 +1131,11 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
@Override
+ protected byte[] getValueBytesFromIdWithoutCache(int id) {
+ throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id");
+ }
+
+ @Override
public int getMinId() {
return baseId;
}
@@ -1149,16 +1156,6 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
- @Override
- protected byte[] getValueBytesFromIdImpl(int id) {
- throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id");
- }
-
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id");
- }
-
public void flushIndex(CachedTreeMap dictSliceMap, boolean keepAppend) throws IOException {
try (FSDataOutputStream indexOut = dictSliceMap.openIndexOutput()) {
indexOut.writeInt(baseId);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index b2bad53..156971d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -52,13 +52,13 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
if (id != null)
return id.intValue();
byte[] valueBytes = bytesConvert.convertToBytes(value);
- id = getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag);
+ id = getIdFromValueBytesWithoutCache(valueBytes, 0, valueBytes.length, roundingFlag);
cache.put(value, id);
return id;
}
}
byte[] valueBytes = bytesConvert.convertToBytes(value);
- return getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag);
+ return getIdFromValueBytesWithoutCache(valueBytes, 0, valueBytes.length, roundingFlag);
}
//id --> value
@@ -70,13 +70,13 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
int seq = calcSeqNoFromId(id);
if (cache[seq] != null)
return (T) cache[seq];
- byte[] valueBytes = getValueBytesFromIdImpl(id);
+ byte[] valueBytes = getValueBytesFromIdWithoutCache(id);
T value = bytesConvert.convertFromBytes(valueBytes, 0, valueBytes.length);
cache[seq] = value;
return value;
}
}
- byte[] valueBytes = getValueBytesFromIdImpl(id);
+ byte[] valueBytes = getValueBytesFromIdWithoutCache(id);
return bytesConvert.convertFromBytes(valueBytes, 0, valueBytes.length);
}
@@ -99,4 +99,9 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
this.valueToIdCache = null;
this.idToValueCache = null;
}
+
+ abstract protected byte[] getValueBytesFromIdWithoutCache(int id);
+
+ abstract protected int getIdFromValueBytesWithoutCache(byte[] valueBytes, int offset, int length, int roundingFlag);
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 29bbee2..56b4994 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -26,7 +26,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
import java.util.Date;
import org.apache.commons.lang.StringUtils;
@@ -84,10 +83,6 @@ public class DateStrDictionary extends Dictionary<String> {
return pattern.length();
}
- @Override
- protected boolean isNullByteForm(byte[] value, int offset, int len) {
- return value == null || len == 0;
- }
@Override
final protected int getIdFromValueImpl(String value, int roundFlag) {
@@ -107,34 +102,6 @@ public class DateStrDictionary extends Dictionary<String> {
return dateToString(new Date(millis), pattern);
}
- @Override
- final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- try {
- return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- @Override
- final protected byte[] getValueBytesFromIdImpl(int id) {
- String date = getValueFromId(id);
- byte[] bytes;
- try {
- bytes = date.getBytes("ISO-8859-1");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- return bytes;
- }
-
- @Override
- final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- byte[] bytes = getValueBytesFromIdImpl(id);
- System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
- return bytes.length;
- }
-
private int calcIdFromSeqNo(int seq) {
return seq < 0 ? seq : baseId + seq;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 6f4f2c4..c23e7b5 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -227,7 +227,7 @@ public class DictionaryGenerator {
@Override
public void init(DictionaryInfo info, int baseId) throws IOException {
this.baseId = baseId;
- this.builder = new NumberDictionaryBuilder(new StringBytesConverter());
+ this.builder = new NumberDictionaryBuilder(new NumberDictionaryForestBuilder.Number2BytesConverter());
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index 9458e9d..c55937d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -175,37 +175,9 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
}
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- NumberBytesCodec codec = getCodec();
- codec.encodeNumber(value, offset, len);
- return super.getIdFromValueBytesImpl(codec.buf, codec.bufOffset, codec.bufLen, roundingFlag);
- }
-
- @Override
protected boolean isNullObjectForm(T value) {
return value == null || value.equals("");
}
+
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- NumberBytesCodec codec = getCodec();
- codec.bufOffset = 0;
- codec.bufLen = super.getValueBytesFromIdImpl(id, codec.buf, 0);
- return codec.decodeNumber(returnValue, offset);
- }
-
-
- public static void main(String[] args) throws Exception {
- NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<String>(new StringBytesConverter());
- b.addValue("10");
- b.addValue("100");
- b.addValue("40");
- b.addValue("7");
- TrieDictionary<String> dict = b.build(0);
-
- //dict.enableIdToValueBytesCache();
- for (int i = 0; i <= dict.getMaxId(); i++) {
- System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index 68a05d4..b9e94414 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -18,7 +18,6 @@
package org.apache.kylin.dict;
-import org.apache.kylin.common.util.Bytes;
/**
* Use <code>NumberDictionaryForestBuilder</code> instead.
@@ -28,18 +27,11 @@ import org.apache.kylin.common.util.Bytes;
@Deprecated
public class NumberDictionaryBuilder<T> extends TrieDictionaryBuilder<T> {
- NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec(NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT);
public NumberDictionaryBuilder(BytesConverter<T> bytesConverter) {
super(bytesConverter);
}
- @Override
- void addValue(byte[] value) {
- codec.encodeNumber(value, 0, value.length);
- byte[] copy = Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen);
- super.addValue(copy);
- }
public NumberDictionary<T> build(int baseId) {
byte[] trieBytes = buildTrieBytes(baseId);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index eabc9f1..d075ce1 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -22,7 +22,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
@@ -85,34 +84,6 @@ public class TimeStrDictionary extends Dictionary<String> {
}
@Override
- final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- try {
- return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- @Override
- final protected byte[] getValueBytesFromIdImpl(int id) {
- String date = getValueFromId(id);
- byte[] bytes;
- try {
- bytes = date.getBytes("ISO-8859-1");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- return bytes;
- }
-
- @Override
- final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- byte[] bytes = getValueBytesFromIdImpl(id);
- System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
- return bytes.length;
- }
-
- @Override
public void dump(PrintStream out) {
out.println(this.toString());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 8e7f5dc..9b84734 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -141,7 +141,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) {
int seq = lookupSeqNoFromValue(headSize, value, offset, offset + len, roundingFlag);
int id = calcIdFromSeqNo(seq);
if (id < 0)
@@ -236,7 +236,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
@Override
- protected byte[] getValueBytesFromIdImpl(int id) {
+ protected byte[] getValueBytesFromIdWithoutCache(int id) {
byte[] buf = new byte[maxValueLength];
int len = getValueBytesFromIdImpl(id, buf, 0);
@@ -249,7 +249,6 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
}
}
- @Override
protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
int seq = calcSeqNoFromId(id);
return lookupValueFromSeqNo(headSize, seq, returnValue, offset);
@@ -425,7 +424,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
//dict2.enableIdToValueBytesCache();
for (int i = 0; i <= dict.getMaxId(); i++) {
- System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
+ System.out.println(Bytes.toString(dict.getValueBytesFromIdWithoutCache(i)));
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 598865b..102c49e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -74,7 +74,7 @@ public class TrieDictionaryBuilder<T> {
// ============================================================================
private Node root;
- private BytesConverter<T> bytesConverter;
+ protected BytesConverter<T> bytesConverter;
public TrieDictionaryBuilder(BytesConverter<T> bytesConverter) {
this.root = new Node(new byte[0], false);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 69b4b9c..d1b5a99 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -104,15 +104,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
-
- int result = _getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- //logger.info("{} => {}, rounding {}", bytesConvert.convertFromBytes(value, offset, len), result, roundingFlag);
- return result;
- }
-
- // id = tree_inner_offset + accumulate_offset + baseId
- protected int _getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
+ protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
int index;
if (trees.size() == 1) {
index = 0;
@@ -136,27 +128,20 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
}
TrieDictionary<T> tree = trees.get(index);
- int id = tree.getIdFromValueBytes(value, offset, len, roundingFlag);
+ int id = tree.getIdFromValueBytesWithoutCache(value, offset, len, roundingFlag);
+ if(id == -1)
+ throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
id = id + accuOffset.get(index);
id += baseId;
return id;
}
@Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) throws IllegalArgumentException {
- int index = (trees.size() == 1) ? 0 : findIndexById(id);
- int treeInnerOffset = getTreeInnerOffset(id, index);
- TrieDictionary<T> tree = trees.get(index);
- int size = tree.getValueBytesFromIdImpl(treeInnerOffset, returnValue, offset);
- return size;
- }
-
- @Override
- protected byte[] getValueBytesFromIdImpl(int id) throws IllegalArgumentException {
+ protected byte[] getValueBytesFromIdWithoutCache(int id) throws IllegalArgumentException {
int index = (trees.size() == 1) ? 0 : findIndexById(id);
int treeInnerOffset = getTreeInnerOffset(id, index);
TrieDictionary<T> tree = trees.get(index);
- byte[] result = tree.getValueBytesFromId(treeInnerOffset);
+ byte[] result = tree.getValueBytesFromIdWithoutCache(treeInnerOffset);
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
index af2e302..69da472 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
@@ -131,7 +131,7 @@ public class TrieDictionaryForestBuilder<T> {
trees.add(tree);
int minId = tree.getMinId();
accuOffset.add(curOffset);
- byte[] valueBytes = tree.getValueBytesFromId(minId);
+ byte[] valueBytes = tree.getValueBytesFromIdWithoutCache(minId);
valueDivide.add(new ByteArray(valueBytes, 0, valueBytes.length));
curOffset += (tree.getMaxId() + 1);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index 921925c..18913d0 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -180,7 +180,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
for (; checkIndex < firstAppend; checkIndex++) {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
checkMap.put(id, str);
@@ -200,7 +200,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
for (; checkIndex < secondAppend; checkIndex++) {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
if (checkIndex < firstAppend) {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
@@ -224,7 +224,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
for (; checkIndex < strList.size(); checkIndex++) {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
if (checkIndex < secondAppend) {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
@@ -237,7 +237,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
if (notfound != null) {
for (String s : notfound) {
byte[] bytes = converter.convertToBytes(s);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertEquals(-1, id);
}
}
@@ -245,7 +245,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
dict = testSerialize(dict, converter);
for (String str : strList) {
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
index ffc8ceb..c6d2e05 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
@@ -66,9 +66,6 @@ public class DateStrDictionaryTest {
public void testNull() {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
index 73e0935..2e90bcf 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -134,20 +134,6 @@ public class MultipleDictionaryValueEnumeratorTest {
return "" + values[id];
}
- @Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- return 0;
- }
-
- @Override
- protected byte[] getValueBytesFromIdImpl(int id) {
- return null;
- }
-
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- return 0;
- }
@Override
public void dump(PrintStream out) {}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 23e925c..38f2648 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -59,7 +59,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
@Test
public void testMinMax() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter());
builder.addValue("" + Long.MAX_VALUE);
builder.addValue("" + Long.MIN_VALUE);
NumberDictionary<String> dict = builder.build(0);
@@ -123,7 +123,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
int n = 100;
Set<BigDecimal> set = Sets.newHashSet();
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter());
for (int i = 0; i < n; i++) {
String num = randNumber();
if (set.add(new BigDecimal(num))) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
index ce07a86..f981942 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
@@ -23,6 +23,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.fail;
+
/**
*/
public class TimeStrDictionaryTest {
@@ -62,7 +64,12 @@ public class TimeStrDictionaryTest {
@Test
public void testIllegal() {
- Assert.assertEquals(-1, dict.getIdFromValue("2038-01-19 03:14:07"));
+ try{
+ dict.getIdFromValue("2038-01-19 03:14:07");
+ fail("should throw exception");
+ }catch (IllegalArgumentException e){
+ //correct
+ }
}
public void encodeDecode(String origin) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
index 0b4c0e3..adc1074 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
@@ -25,7 +25,6 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.List;
import java.util.Random;
import java.util.UUID;
@@ -123,7 +122,8 @@ public class TrieDictionaryForestBenchmark {
int step = 1;
for (int i = 0; i < testTimes; i++) {
for (int j = 0; j < cardnality; j++) {
- step |= dict.getValueBytesFromId(j).length;
+ //step |= dict.getValueBytesFromId(j).length;
+ step |= dict.getValueFromId(j).length();
}
}
return System.currentTimeMillis() - startTime;
@@ -135,8 +135,7 @@ public class TrieDictionaryForestBenchmark {
byte[] returnValue = new byte[2048];
for (int i = 0; i < testTimes; i++) {
for (int j = 0; j < cardnality; j++) {
- int size = dict.getValueBytesFromId(j, returnValue, 0);
- step |= size;
+ step |= dict.getValueFromId(j).length();
}
}
return System.currentTimeMillis() - startTime;
@@ -154,16 +153,11 @@ public class TrieDictionaryForestBenchmark {
}
private long runQueryIdByValueBytes(ArrayList<String> rawData, Dictionary<String> dict, int cardnality, int testTimes) {
- List<byte[]> testBytes = new ArrayList<>();
- StringBytesConverter converter = new StringBytesConverter();
- for (int i = 0; i < cardnality; i++) {
- testBytes.add(converter.convertToBytes(rawData.get(i)));
- }
long startTime = System.currentTimeMillis();
int step = 1;
for (int i = 0; i < testTimes; i++) {
for (int j = 0; j < cardnality; j++) {
- step |= dict.getIdFromValueBytes(testBytes.get(j), 0, testBytes.get(j).length);
+ step |= dict.getIdFromValue(rawData.get(j));
}
}
return System.currentTimeMillis() - startTime;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 68cf301..c7fb9c4 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -641,7 +641,7 @@ public class TrieDictionaryForestTest {
System.out.println("max memory:" + Runtime.getRuntime().maxMemory());
System.gc();
Thread.currentThread().sleep(1000);
- NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
int k = 0;
while (true) {
b.addValue(k + "");
@@ -820,7 +820,7 @@ public class TrieDictionaryForestTest {
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
//System.out.println("looking for value:"+new String(array[j]));
- keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length);
+ keep |= dict.getIdFromValueBytesWithoutCache(array[j], 0, array[j].length, 0);
}
}
long timeValueToIdByDict = System.currentTimeMillis() - start;
@@ -845,7 +845,7 @@ public class TrieDictionaryForestTest {
start = System.currentTimeMillis();
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
- keep |= dict.getValueBytesFromId(j, valueBytes, 0);
+ keep |= dict.getValueBytesFromIdWithoutCache(j).length;
}
}
long timeIdToValueByDict = System.currentTimeMillis() - start;
@@ -884,7 +884,7 @@ public class TrieDictionaryForestTest {
for (String s : notFound) {
try {
int nullId = dict.getIdFromValue(s);
- //System.out.println("null value id:" + nullId);
+ System.out.println("null value id:" + nullId);
fail("For not found value '" + s + "', IllegalArgumentException is expected");
} catch (IllegalArgumentException e) {
// good
@@ -905,9 +905,6 @@ public class TrieDictionaryForestTest {
// test null value
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
}
private Map<String, Integer> rightIdMap(int baseId, ArrayList<String> strs) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index a87d7cb..22a93a0 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -47,7 +47,7 @@ public class TrieDictionaryTest {
int count = (int) (Integer.MAX_VALUE * 0.8 / 64);
benchmarkStringDictionary(new RandomStrings(count));
}
-
+
private static class RandomStrings implements Iterable<String> {
final private int size;
@@ -55,7 +55,7 @@ public class TrieDictionaryTest {
this.size = size;
System.out.println("size = " + size);
}
-
+
@Override
public Iterator<String> iterator() {
return new Iterator<String>() {
@@ -71,11 +71,11 @@ public class TrieDictionaryTest {
public String next() {
if (hasNext() == false)
throw new NoSuchElementException();
-
+
i++;
if (i % 1000000 == 0)
System.out.println(i);
-
+
return nextString();
}
@@ -225,7 +225,7 @@ public class TrieDictionaryTest {
TrieDictionaryBuilder<String> b = newDictBuilder(str);
b.stats().print();
TrieDictionary<String> dict = b.build(0);
-
+
TreeSet<String> set = new TreeSet<String>();
for (String s : str) {
set.add(s);
@@ -282,7 +282,7 @@ public class TrieDictionaryTest {
start = System.currentTimeMillis();
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
- keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length);
+ keep |= dict.getIdFromValueBytesWithoutCache(array[j], 0, array[j].length, 0);
}
}
long timeValueToIdByDict = System.currentTimeMillis() - start;
@@ -304,12 +304,12 @@ public class TrieDictionaryTest {
start = System.currentTimeMillis();
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
- keep |= dict.getValueBytesFromId(j, valueBytes, 0);
+ keep |= dict.getValueBytesFromIdWithoutCache(j).length;
}
}
long timeIdToValueByDict = System.currentTimeMillis() - start;
System.out.println(timeIdToValueByDict);
-
+
return keep;
}
@@ -350,9 +350,6 @@ public class TrieDictionaryTest {
// test null value
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
}
private static TrieDictionary<String> testSerialize(TrieDictionary<String> dict) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index b6f1975..de2a34a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -18,6 +18,8 @@
package org.apache.kylin.measure.bitmap;
+import static com.google.common.base.Preconditions.checkArgument;
+
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -38,8 +40,6 @@ import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
import com.google.common.collect.ImmutableMap;
-import static com.google.common.base.Preconditions.checkArgument;
-
/**
* Created by sunyerui on 15/12/10.
*/
@@ -133,14 +133,13 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
Dictionary<String> mergedDict = newDicts.get(colRef);
MutableBitmapCounter retValue = new MutableBitmapCounter();
- byte[] literal = new byte[sourceDict.getSizeOfValue()];
for (int id : value) {
int newId;
- int size = sourceDict.getValueBytesFromId(id, literal, 0);
- if (size < 0) {
+ String v = sourceDict.getValueFromId(id);
+ if (v == null) {
newId = mergedDict.nullId();
} else {
- newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+ newId = mergedDict.getIdFromValue(v);
}
retValue.add(newId);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index a5bb06b..3b6e659 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -133,17 +133,16 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
int valueSize = value.size();
byte[] newIdBuf = new byte[valueSize * mergedDict.getSizeOfId()];
- byte[] literal = new byte[sourceDict.getSizeOfValue()];
int bufOffset = 0;
for (ByteArray c : value) {
int oldId = BytesUtil.readUnsigned(c.array(), c.offset(), c.length());
int newId;
- int size = sourceDict.getValueBytesFromId(oldId, literal, 0);
- if (size < 0) {
+ String v = sourceDict.getValueFromId(oldId);
+ if (v == null) {
newId = mergedDict.nullId();
} else {
- newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+ newId = mergedDict.getIdFromValue(v);
}
BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId());
c.set(newIdBuf, bufOffset, mergedDict.getSizeOfId());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index fcd0182..ae186e2 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.dimension.DictionaryDimEnc;
@@ -601,7 +602,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
@SuppressWarnings("rawtypes")
private static Dictionary newDictionaryOfInteger() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
builder.addValue("10");
builder.addValue("20");
builder.addValue("30");
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index a79d5aa..047e2b1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -186,8 +186,8 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
if (this.checkNeedMerging(col)) {
// if dictionary on fact table column, needs rewrite
DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
- Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
+ Dictionary<String> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
+ Dictionary<String> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
while (sourceDict.getSizeOfValue() > newKeyBodyBuf.length - bufOffset || //
mergedDict.getSizeOfValue() > newKeyBodyBuf.length - bufOffset || //
@@ -200,11 +200,12 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[useSplit].value, 0, splittedByteses[useSplit].length);
int idInMergedDict;
- int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset);
- if (size < 0) {
+ //int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset);
+ String v = sourceDict.getValueFromId(idInSourceDict);
+ if (v == null) {
idInMergedDict = mergedDict.nullId();
} else {
- idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBodyBuf, bufOffset, size);
+ idInMergedDict = mergedDict.getIdFromValue(v);
}
BytesUtil.writeUnsigned(idInMergedDict, newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index f6664bf..72793f5 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -38,7 +38,6 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.dict.NumberDictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
import org.apache.kylin.dict.NumberDictionaryForestBuilder;
-import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryForest;
import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
import org.junit.Ignore;
@@ -121,7 +120,7 @@ public class NumberDictionaryForestTest {
TrieDictionaryForest<String> dict = b.build();
dict.dump(System.out);
- NumberDictionaryBuilder<String> b2 = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> b2 = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
for (String str : testData)
b2.addValue(str);
NumberDictionary<String> dict2 = b2.build(0);
@@ -148,22 +147,20 @@ public class NumberDictionaryForestTest {
assertTrue(dict1.getSizeOfId() == dict2.getSizeOfId());
assertTrue(dict1.getSizeOfValue() == dict2.getSizeOfValue());
-
+
byte[] buf = new byte[dict1.getSizeOfValue()];
-
+
{
- int len = dict1.getValueBytesFromId(0, buf, 0);
- int newId = dict2.getIdFromValueBytes(buf, 0, len);
+ int newId = dict2.getIdFromValue(dict1.getValueFromId(0));
assertTrue(newId == 0);
}
{
- int len = dict1.getValueBytesFromId(1, buf, 0);
- int newId = dict2.getIdFromValueBytes(buf, 0, len);
+
+ int newId = dict2.getIdFromValue(dict1.getValueFromId(1));
assertTrue(newId == 2);
}
{
- int len = dict1.getValueBytesFromId(2, buf, 0);
- int newId = dict2.getIdFromValueBytes(buf, 0, len);
+ int newId = dict2.getIdFromValue(dict1.getValueFromId(2));
assertTrue(newId == 4);
}
}