You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/01/17 03:06:53 UTC
[1/2] kylin git commit: KYLIN-2400 remove byte[] interface in
dictionary
Repository: kylin
Updated Branches:
refs/heads/master e20e2b2e8 -> 576d2dd35
KYLIN-2400 remove byte[] interface in dictionary
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e769bfa0
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e769bfa0
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e769bfa0
Branch: refs/heads/master
Commit: e769bfa004e9d548c5a042f02765cc093ce70ec1
Parents: e20e2b2
Author: xiefan46 <95...@qq.com>
Authored: Mon Jan 9 19:11:39 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Jan 17 10:54:22 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/Dictionary.java | 46 ++++----------------
.../apache/kylin/dict/AppendTrieDictionary.java | 19 ++++----
.../org/apache/kylin/dict/CacheDictionary.java | 13 ++++--
.../apache/kylin/dict/DateStrDictionary.java | 33 --------------
.../apache/kylin/dict/DictionaryGenerator.java | 2 +-
.../org/apache/kylin/dict/NumberDictionary.java | 30 +------------
.../kylin/dict/NumberDictionaryBuilder.java | 8 ----
.../apache/kylin/dict/TimeStrDictionary.java | 29 ------------
.../org/apache/kylin/dict/TrieDictionary.java | 7 ++-
.../kylin/dict/TrieDictionaryBuilder.java | 2 +-
.../apache/kylin/dict/TrieDictionaryForest.java | 27 +++---------
.../kylin/dict/TrieDictionaryForestBuilder.java | 2 +-
.../kylin/dict/AppendTrieDictionaryTest.java | 10 ++---
.../kylin/dict/DateStrDictionaryTest.java | 3 --
.../MultipleDictionaryValueEnumeratorTest.java | 14 ------
.../apache/kylin/dict/NumberDictionaryTest.java | 4 +-
.../kylin/dict/TimeStrDictionaryTest.java | 9 +++-
.../dict/TrieDictionaryForestBenchmark.java | 14 ++----
.../kylin/dict/TrieDictionaryForestTest.java | 11 ++---
.../apache/kylin/dict/TrieDictionaryTest.java | 19 ++++----
.../kylin/measure/bitmap/BitmapMeasureType.java | 11 +++--
.../kylin/measure/raw/RawMeasureType.java | 7 ++-
.../storage/gtrecord/DictGridTableTest.java | 3 +-
.../engine/mr/steps/MergeCuboidMapper.java | 11 ++---
.../mr/steps/NumberDictionaryForestTest.java | 17 +++-----
25 files changed, 92 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 1e172bc..9d675f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -94,8 +94,13 @@ abstract public class Dictionary<T> implements Serializable {
final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException {
if (isNullObjectForm(value))
return nullId();
- else
- return getIdFromValueImpl(value, roundingFlag);
+ else {
+ int id = getIdFromValueImpl(value, roundingFlag);
+ if(id == -1){
+ throw new IllegalArgumentException("Value : " + value + "no exists");
+ }
+ return id;
+ }
}
final public boolean containsValue(T value) throws IllegalArgumentException {
@@ -136,9 +141,7 @@ abstract public class Dictionary<T> implements Serializable {
* Convenient form of
* <code>getIdFromValueBytes(value, offset, len, 0)</code>
*/
- final public int getIdFromValueBytes(byte[] value, int offset, int len) throws IllegalArgumentException {
- return getIdFromValueBytes(value, offset, len, 0);
- }
+
/**
* A lower level API, return ID integer from raw value bytes. In case of not found
@@ -153,31 +156,6 @@ abstract public class Dictionary<T> implements Serializable {
* if value is not found in dictionary and rounding is off;
* or if rounding cannot find a smaller or bigger ID
*/
- final public int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
- if (isNullByteForm(value, offset, len))
- return nullId();
- else {
- int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- if (id == -1)
- throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
- return id;
- }
- }
-
- protected boolean isNullByteForm(byte[] value, int offset, int len) {
- return value == null;
- }
-
- abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
-
- final public byte[] getValueBytesFromId(int id) {
- if (isNullId(id))
- return BytesUtil.EMPTY_BYTE_ARRAY;
- else
- return getValueBytesFromIdImpl(id);
- }
-
- abstract protected byte[] getValueBytesFromIdImpl(int id);
/**
* A lower level API, get byte values from ID, return the number of bytes
@@ -189,14 +167,6 @@ abstract public class Dictionary<T> implements Serializable {
* @throws IllegalArgumentException
* if ID is not found in dictionary
*/
- final public int getValueBytesFromId(int id, byte[] returnValue, int offset) throws IllegalArgumentException {
- if (isNullId(id))
- return -1;
- else
- return getValueBytesFromIdImpl(id, returnValue, offset);
- }
-
- abstract protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset);
abstract public void dump(PrintStream out);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index b797167..3aef967 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -97,6 +97,8 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
enableCache();
}
+
+
public void initParams(String baseDir, int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter) throws IOException {
this.baseDir = baseDir;
this.baseId = baseId;
@@ -1113,7 +1115,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) {
if (dictSliceMap.isEmpty()) {
return -1;
}
@@ -1129,6 +1131,11 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
@Override
+ protected byte[] getValueBytesFromIdWithoutCache(int id) {
+ throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id");
+ }
+
+ @Override
public int getMinId() {
return baseId;
}
@@ -1149,16 +1156,6 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
- @Override
- protected byte[] getValueBytesFromIdImpl(int id) {
- throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id");
- }
-
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- throw new UnsupportedOperationException("AppendTrieDictionary can't retrive value from id");
- }
-
public void flushIndex(CachedTreeMap dictSliceMap, boolean keepAppend) throws IOException {
try (FSDataOutputStream indexOut = dictSliceMap.openIndexOutput()) {
indexOut.writeInt(baseId);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
index b2bad53..156971d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/CacheDictionary.java
@@ -52,13 +52,13 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
if (id != null)
return id.intValue();
byte[] valueBytes = bytesConvert.convertToBytes(value);
- id = getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag);
+ id = getIdFromValueBytesWithoutCache(valueBytes, 0, valueBytes.length, roundingFlag);
cache.put(value, id);
return id;
}
}
byte[] valueBytes = bytesConvert.convertToBytes(value);
- return getIdFromValueBytes(valueBytes, 0, valueBytes.length, roundingFlag);
+ return getIdFromValueBytesWithoutCache(valueBytes, 0, valueBytes.length, roundingFlag);
}
//id --> value
@@ -70,13 +70,13 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
int seq = calcSeqNoFromId(id);
if (cache[seq] != null)
return (T) cache[seq];
- byte[] valueBytes = getValueBytesFromIdImpl(id);
+ byte[] valueBytes = getValueBytesFromIdWithoutCache(id);
T value = bytesConvert.convertFromBytes(valueBytes, 0, valueBytes.length);
cache[seq] = value;
return value;
}
}
- byte[] valueBytes = getValueBytesFromIdImpl(id);
+ byte[] valueBytes = getValueBytesFromIdWithoutCache(id);
return bytesConvert.convertFromBytes(valueBytes, 0, valueBytes.length);
}
@@ -99,4 +99,9 @@ public abstract class CacheDictionary<T> extends Dictionary<T> {
this.valueToIdCache = null;
this.idToValueCache = null;
}
+
+ abstract protected byte[] getValueBytesFromIdWithoutCache(int id);
+
+ abstract protected int getIdFromValueBytesWithoutCache(byte[] valueBytes, int offset, int length, int roundingFlag);
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 29bbee2..56b4994 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -26,7 +26,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
import java.util.Date;
import org.apache.commons.lang.StringUtils;
@@ -84,10 +83,6 @@ public class DateStrDictionary extends Dictionary<String> {
return pattern.length();
}
- @Override
- protected boolean isNullByteForm(byte[] value, int offset, int len) {
- return value == null || len == 0;
- }
@Override
final protected int getIdFromValueImpl(String value, int roundFlag) {
@@ -107,34 +102,6 @@ public class DateStrDictionary extends Dictionary<String> {
return dateToString(new Date(millis), pattern);
}
- @Override
- final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- try {
- return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- @Override
- final protected byte[] getValueBytesFromIdImpl(int id) {
- String date = getValueFromId(id);
- byte[] bytes;
- try {
- bytes = date.getBytes("ISO-8859-1");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- return bytes;
- }
-
- @Override
- final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- byte[] bytes = getValueBytesFromIdImpl(id);
- System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
- return bytes.length;
- }
-
private int calcIdFromSeqNo(int seq) {
return seq < 0 ? seq : baseId + seq;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 6f4f2c4..c23e7b5 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -227,7 +227,7 @@ public class DictionaryGenerator {
@Override
public void init(DictionaryInfo info, int baseId) throws IOException {
this.baseId = baseId;
- this.builder = new NumberDictionaryBuilder(new StringBytesConverter());
+ this.builder = new NumberDictionaryBuilder(new NumberDictionaryForestBuilder.Number2BytesConverter());
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index 9458e9d..c55937d 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -175,37 +175,9 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
}
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- NumberBytesCodec codec = getCodec();
- codec.encodeNumber(value, offset, len);
- return super.getIdFromValueBytesImpl(codec.buf, codec.bufOffset, codec.bufLen, roundingFlag);
- }
-
- @Override
protected boolean isNullObjectForm(T value) {
return value == null || value.equals("");
}
+
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- NumberBytesCodec codec = getCodec();
- codec.bufOffset = 0;
- codec.bufLen = super.getValueBytesFromIdImpl(id, codec.buf, 0);
- return codec.decodeNumber(returnValue, offset);
- }
-
-
- public static void main(String[] args) throws Exception {
- NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<String>(new StringBytesConverter());
- b.addValue("10");
- b.addValue("100");
- b.addValue("40");
- b.addValue("7");
- TrieDictionary<String> dict = b.build(0);
-
- //dict.enableIdToValueBytesCache();
- for (int i = 0; i <= dict.getMaxId(); i++) {
- System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
- }
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index 68a05d4..b9e94414 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -18,7 +18,6 @@
package org.apache.kylin.dict;
-import org.apache.kylin.common.util.Bytes;
/**
* Use <code>NumberDictionaryForestBuilder</code> instead.
@@ -28,18 +27,11 @@ import org.apache.kylin.common.util.Bytes;
@Deprecated
public class NumberDictionaryBuilder<T> extends TrieDictionaryBuilder<T> {
- NumberDictionary.NumberBytesCodec codec = new NumberDictionary.NumberBytesCodec(NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT);
public NumberDictionaryBuilder(BytesConverter<T> bytesConverter) {
super(bytesConverter);
}
- @Override
- void addValue(byte[] value) {
- codec.encodeNumber(value, 0, value.length);
- byte[] copy = Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen);
- super.addValue(copy);
- }
public NumberDictionary<T> build(int baseId) {
byte[] trieBytes = buildTrieBytes(baseId);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
index eabc9f1..d075ce1 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TimeStrDictionary.java
@@ -22,7 +22,6 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
@@ -85,34 +84,6 @@ public class TimeStrDictionary extends Dictionary<String> {
}
@Override
- final protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- try {
- return getIdFromValue(new String(value, offset, len, "ISO-8859-1"));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- }
-
- @Override
- final protected byte[] getValueBytesFromIdImpl(int id) {
- String date = getValueFromId(id);
- byte[] bytes;
- try {
- bytes = date.getBytes("ISO-8859-1");
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException(e); // never happen
- }
- return bytes;
- }
-
- @Override
- final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- byte[] bytes = getValueBytesFromIdImpl(id);
- System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
- return bytes.length;
- }
-
- @Override
public void dump(PrintStream out) {
out.println(this.toString());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index 8e7f5dc..9b84734 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -141,7 +141,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
+ protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) {
int seq = lookupSeqNoFromValue(headSize, value, offset, offset + len, roundingFlag);
int id = calcIdFromSeqNo(seq);
if (id < 0)
@@ -236,7 +236,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
@Override
- protected byte[] getValueBytesFromIdImpl(int id) {
+ protected byte[] getValueBytesFromIdWithoutCache(int id) {
byte[] buf = new byte[maxValueLength];
int len = getValueBytesFromIdImpl(id, buf, 0);
@@ -249,7 +249,6 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
}
}
- @Override
protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
int seq = calcSeqNoFromId(id);
return lookupValueFromSeqNo(headSize, seq, returnValue, offset);
@@ -425,7 +424,7 @@ public class TrieDictionary<T> extends CacheDictionary<T> {
//dict2.enableIdToValueBytesCache();
for (int i = 0; i <= dict.getMaxId(); i++) {
- System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
+ System.out.println(Bytes.toString(dict.getValueBytesFromIdWithoutCache(i)));
}
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 598865b..102c49e 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -74,7 +74,7 @@ public class TrieDictionaryBuilder<T> {
// ============================================================================
private Node root;
- private BytesConverter<T> bytesConverter;
+ protected BytesConverter<T> bytesConverter;
public TrieDictionaryBuilder(BytesConverter<T> bytesConverter) {
this.root = new Node(new byte[0], false);
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 69b4b9c..d1b5a99 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -104,15 +104,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
@Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
-
- int result = _getIdFromValueBytesImpl(value, offset, len, roundingFlag);
- //logger.info("{} => {}, rounding {}", bytesConvert.convertFromBytes(value, offset, len), result, roundingFlag);
- return result;
- }
-
- // id = tree_inner_offset + accumulate_offset + baseId
- protected int _getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
+ protected int getIdFromValueBytesWithoutCache(byte[] value, int offset, int len, int roundingFlag) throws IllegalArgumentException {
int index;
if (trees.size() == 1) {
index = 0;
@@ -136,27 +128,20 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
}
TrieDictionary<T> tree = trees.get(index);
- int id = tree.getIdFromValueBytes(value, offset, len, roundingFlag);
+ int id = tree.getIdFromValueBytesWithoutCache(value, offset, len, roundingFlag);
+ if(id == -1)
+ throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
id = id + accuOffset.get(index);
id += baseId;
return id;
}
@Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) throws IllegalArgumentException {
- int index = (trees.size() == 1) ? 0 : findIndexById(id);
- int treeInnerOffset = getTreeInnerOffset(id, index);
- TrieDictionary<T> tree = trees.get(index);
- int size = tree.getValueBytesFromIdImpl(treeInnerOffset, returnValue, offset);
- return size;
- }
-
- @Override
- protected byte[] getValueBytesFromIdImpl(int id) throws IllegalArgumentException {
+ protected byte[] getValueBytesFromIdWithoutCache(int id) throws IllegalArgumentException {
int index = (trees.size() == 1) ? 0 : findIndexById(id);
int treeInnerOffset = getTreeInnerOffset(id, index);
TrieDictionary<T> tree = trees.get(index);
- byte[] result = tree.getValueBytesFromId(treeInnerOffset);
+ byte[] result = tree.getValueBytesFromIdWithoutCache(treeInnerOffset);
return result;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
index af2e302..69da472 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
@@ -131,7 +131,7 @@ public class TrieDictionaryForestBuilder<T> {
trees.add(tree);
int minId = tree.getMinId();
accuOffset.add(curOffset);
- byte[] valueBytes = tree.getValueBytesFromId(minId);
+ byte[] valueBytes = tree.getValueBytesFromIdWithoutCache(minId);
valueDivide.add(new ByteArray(valueBytes, 0, valueBytes.length));
curOffset += (tree.getMaxId() + 1);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index 921925c..18913d0 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -180,7 +180,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
for (; checkIndex < firstAppend; checkIndex++) {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
assertFalse(String.format("Id %d for %s should be empty, but is %s", id, str, checkMap.get(id)), checkMap.containsKey(id) && !str.equals(checkMap.get(id)));
checkMap.put(id, str);
@@ -200,7 +200,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
for (; checkIndex < secondAppend; checkIndex++) {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
if (checkIndex < firstAppend) {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
@@ -224,7 +224,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
for (; checkIndex < strList.size(); checkIndex++) {
String str = strList.get(checkIndex);
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
if (checkIndex < secondAppend) {
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
@@ -237,7 +237,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
if (notfound != null) {
for (String s : notfound) {
byte[] bytes = converter.convertToBytes(s);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertEquals(-1, id);
}
}
@@ -245,7 +245,7 @@ public class AppendTrieDictionaryTest extends LocalFileMetadataTestCase {
dict = testSerialize(dict, converter);
for (String str : strList) {
byte[] bytes = converter.convertToBytes(str);
- int id = dict.getIdFromValueBytesImpl(bytes, 0, bytes.length, 0);
+ int id = dict.getIdFromValueBytesWithoutCache(bytes, 0, bytes.length, 0);
assertNotEquals(String.format("Value %s not exist", str), -1, id);
assertEquals("Except id " + id + " for " + str + " but " + checkMap.get(id), str, checkMap.get(id));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
index ffc8ceb..c6d2e05 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/DateStrDictionaryTest.java
@@ -66,9 +66,6 @@ public class DateStrDictionaryTest {
public void testNull() {
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
}
@Test
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
index 73e0935..2e90bcf 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/MultipleDictionaryValueEnumeratorTest.java
@@ -134,20 +134,6 @@ public class MultipleDictionaryValueEnumeratorTest {
return "" + values[id];
}
- @Override
- protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag) {
- return 0;
- }
-
- @Override
- protected byte[] getValueBytesFromIdImpl(int id) {
- return null;
- }
-
- @Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- return 0;
- }
@Override
public void dump(PrintStream out) {}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 23e925c..38f2648 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -59,7 +59,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
@Test
public void testMinMax() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter());
builder.addValue("" + Long.MAX_VALUE);
builder.addValue("" + Long.MIN_VALUE);
NumberDictionary<String> dict = builder.build(0);
@@ -123,7 +123,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
int n = 100;
Set<BigDecimal> set = Sets.newHashSet();
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter());
for (int i = 0; i < n; i++) {
String num = randNumber();
if (set.add(new BigDecimal(num))) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
index ce07a86..f981942 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
@@ -23,6 +23,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.fail;
+
/**
*/
public class TimeStrDictionaryTest {
@@ -62,7 +64,12 @@ public class TimeStrDictionaryTest {
@Test
public void testIllegal() {
- Assert.assertEquals(-1, dict.getIdFromValue("2038-01-19 03:14:07"));
+ try{
+ dict.getIdFromValue("2038-01-19 03:14:07");
+ fail("should throw exception");
+ }catch (IllegalArgumentException e){
+ //correct
+ }
}
public void encodeDecode(String origin) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
index 0b4c0e3..adc1074 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestBenchmark.java
@@ -25,7 +25,6 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.List;
import java.util.Random;
import java.util.UUID;
@@ -123,7 +122,8 @@ public class TrieDictionaryForestBenchmark {
int step = 1;
for (int i = 0; i < testTimes; i++) {
for (int j = 0; j < cardnality; j++) {
- step |= dict.getValueBytesFromId(j).length;
+ //step |= dict.getValueBytesFromId(j).length;
+ step |= dict.getValueFromId(j).length();
}
}
return System.currentTimeMillis() - startTime;
@@ -135,8 +135,7 @@ public class TrieDictionaryForestBenchmark {
byte[] returnValue = new byte[2048];
for (int i = 0; i < testTimes; i++) {
for (int j = 0; j < cardnality; j++) {
- int size = dict.getValueBytesFromId(j, returnValue, 0);
- step |= size;
+ step |= dict.getValueFromId(j).length();
}
}
return System.currentTimeMillis() - startTime;
@@ -154,16 +153,11 @@ public class TrieDictionaryForestBenchmark {
}
private long runQueryIdByValueBytes(ArrayList<String> rawData, Dictionary<String> dict, int cardnality, int testTimes) {
- List<byte[]> testBytes = new ArrayList<>();
- StringBytesConverter converter = new StringBytesConverter();
- for (int i = 0; i < cardnality; i++) {
- testBytes.add(converter.convertToBytes(rawData.get(i)));
- }
long startTime = System.currentTimeMillis();
int step = 1;
for (int i = 0; i < testTimes; i++) {
for (int j = 0; j < cardnality; j++) {
- step |= dict.getIdFromValueBytes(testBytes.get(j), 0, testBytes.get(j).length);
+ step |= dict.getIdFromValue(rawData.get(j));
}
}
return System.currentTimeMillis() - startTime;
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index 68cf301..c7fb9c4 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -641,7 +641,7 @@ public class TrieDictionaryForestTest {
System.out.println("max memory:" + Runtime.getRuntime().maxMemory());
System.gc();
Thread.currentThread().sleep(1000);
- NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
int k = 0;
while (true) {
b.addValue(k + "");
@@ -820,7 +820,7 @@ public class TrieDictionaryForestTest {
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
//System.out.println("looking for value:"+new String(array[j]));
- keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length);
+ keep |= dict.getIdFromValueBytesWithoutCache(array[j], 0, array[j].length, 0);
}
}
long timeValueToIdByDict = System.currentTimeMillis() - start;
@@ -845,7 +845,7 @@ public class TrieDictionaryForestTest {
start = System.currentTimeMillis();
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
- keep |= dict.getValueBytesFromId(j, valueBytes, 0);
+ keep |= dict.getValueBytesFromIdWithoutCache(j).length;
}
}
long timeIdToValueByDict = System.currentTimeMillis() - start;
@@ -884,7 +884,7 @@ public class TrieDictionaryForestTest {
for (String s : notFound) {
try {
int nullId = dict.getIdFromValue(s);
- //System.out.println("null value id:" + nullId);
+ System.out.println("null value id:" + nullId);
fail("For not found value '" + s + "', IllegalArgumentException is expected");
} catch (IllegalArgumentException e) {
// good
@@ -905,9 +905,6 @@ public class TrieDictionaryForestTest {
// test null value
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
}
private Map<String, Integer> rightIdMap(int baseId, ArrayList<String> strs) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index a87d7cb..22a93a0 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -47,7 +47,7 @@ public class TrieDictionaryTest {
int count = (int) (Integer.MAX_VALUE * 0.8 / 64);
benchmarkStringDictionary(new RandomStrings(count));
}
-
+
private static class RandomStrings implements Iterable<String> {
final private int size;
@@ -55,7 +55,7 @@ public class TrieDictionaryTest {
this.size = size;
System.out.println("size = " + size);
}
-
+
@Override
public Iterator<String> iterator() {
return new Iterator<String>() {
@@ -71,11 +71,11 @@ public class TrieDictionaryTest {
public String next() {
if (hasNext() == false)
throw new NoSuchElementException();
-
+
i++;
if (i % 1000000 == 0)
System.out.println(i);
-
+
return nextString();
}
@@ -225,7 +225,7 @@ public class TrieDictionaryTest {
TrieDictionaryBuilder<String> b = newDictBuilder(str);
b.stats().print();
TrieDictionary<String> dict = b.build(0);
-
+
TreeSet<String> set = new TreeSet<String>();
for (String s : str) {
set.add(s);
@@ -282,7 +282,7 @@ public class TrieDictionaryTest {
start = System.currentTimeMillis();
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
- keep |= dict.getIdFromValueBytes(array[j], 0, array[j].length);
+ keep |= dict.getIdFromValueBytesWithoutCache(array[j], 0, array[j].length, 0);
}
}
long timeValueToIdByDict = System.currentTimeMillis() - start;
@@ -304,12 +304,12 @@ public class TrieDictionaryTest {
start = System.currentTimeMillis();
for (int i = 0; i < times; i++) {
for (int j = 0; j < n; j++) {
- keep |= dict.getValueBytesFromId(j, valueBytes, 0);
+ keep |= dict.getValueBytesFromIdWithoutCache(j).length;
}
}
long timeIdToValueByDict = System.currentTimeMillis() - start;
System.out.println(timeIdToValueByDict);
-
+
return keep;
}
@@ -350,9 +350,6 @@ public class TrieDictionaryTest {
// test null value
int nullId = dict.getIdFromValue(null);
assertNull(dict.getValueFromId(nullId));
- int nullId2 = dict.getIdFromValueBytes(null, 0, 0);
- assertEquals(dict.getValueBytesFromId(nullId2, null, 0), -1);
- assertEquals(nullId, nullId2);
}
private static TrieDictionary<String> testSerialize(TrieDictionary<String> dict) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
index b6f1975..de2a34a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/bitmap/BitmapMeasureType.java
@@ -18,6 +18,8 @@
package org.apache.kylin.measure.bitmap;
+import static com.google.common.base.Preconditions.checkArgument;
+
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -38,8 +40,6 @@ import org.apache.kylin.metadata.realization.SQLDigest.SQLCall;
import com.google.common.collect.ImmutableMap;
-import static com.google.common.base.Preconditions.checkArgument;
-
/**
* Created by sunyerui on 15/12/10.
*/
@@ -133,14 +133,13 @@ public class BitmapMeasureType extends MeasureType<BitmapCounter> {
Dictionary<String> mergedDict = newDicts.get(colRef);
MutableBitmapCounter retValue = new MutableBitmapCounter();
- byte[] literal = new byte[sourceDict.getSizeOfValue()];
for (int id : value) {
int newId;
- int size = sourceDict.getValueBytesFromId(id, literal, 0);
- if (size < 0) {
+ String v = sourceDict.getValueFromId(id);
+ if (v == null) {
newId = mergedDict.nullId();
} else {
- newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+ newId = mergedDict.getIdFromValue(v);
}
retValue.add(newId);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index a5bb06b..3b6e659 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -133,17 +133,16 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
int valueSize = value.size();
byte[] newIdBuf = new byte[valueSize * mergedDict.getSizeOfId()];
- byte[] literal = new byte[sourceDict.getSizeOfValue()];
int bufOffset = 0;
for (ByteArray c : value) {
int oldId = BytesUtil.readUnsigned(c.array(), c.offset(), c.length());
int newId;
- int size = sourceDict.getValueBytesFromId(oldId, literal, 0);
- if (size < 0) {
+ String v = sourceDict.getValueFromId(oldId);
+ if (v == null) {
newId = mergedDict.nullId();
} else {
- newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+ newId = mergedDict.getIdFromValue(v);
}
BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId());
c.set(newIdBuf, bufOffset, mergedDict.getSizeOfId());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index fcd0182..ae186e2 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -35,6 +35,7 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.dict.NumberDictionaryBuilder;
+import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.dimension.DictionaryDimEnc;
@@ -601,7 +602,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
@SuppressWarnings("rawtypes")
private static Dictionary newDictionaryOfInteger() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
builder.addValue("10");
builder.addValue("20");
builder.addValue("30");
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
index a79d5aa..047e2b1 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeCuboidMapper.java
@@ -186,8 +186,8 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
if (this.checkNeedMerging(col)) {
// if dictionary on fact table column, needs rewrite
DictionaryManager dictMgr = DictionaryManager.getInstance(config);
- Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
- Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
+ Dictionary<String> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col));
+ Dictionary<String> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
while (sourceDict.getSizeOfValue() > newKeyBodyBuf.length - bufOffset || //
mergedDict.getSizeOfValue() > newKeyBodyBuf.length - bufOffset || //
@@ -200,11 +200,12 @@ public class MergeCuboidMapper extends KylinMapper<Text, Text, Text, Text> {
int idInSourceDict = BytesUtil.readUnsigned(splittedByteses[useSplit].value, 0, splittedByteses[useSplit].length);
int idInMergedDict;
- int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset);
- if (size < 0) {
+ //int size = sourceDict.getValueBytesFromId(idInSourceDict, newKeyBodyBuf, bufOffset);
+ String v = sourceDict.getValueFromId(idInSourceDict);
+ if (v == null) {
idInMergedDict = mergedDict.nullId();
} else {
- idInMergedDict = mergedDict.getIdFromValueBytes(newKeyBodyBuf, bufOffset, size);
+ idInMergedDict = mergedDict.getIdFromValue(v);
}
BytesUtil.writeUnsigned(idInMergedDict, newKeyBodyBuf, bufOffset, mergedDict.getSizeOfId());
http://git-wip-us.apache.org/repos/asf/kylin/blob/e769bfa0/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index f6664bf..72793f5 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -38,7 +38,6 @@ import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.dict.NumberDictionary;
import org.apache.kylin.dict.NumberDictionaryBuilder;
import org.apache.kylin.dict.NumberDictionaryForestBuilder;
-import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryForest;
import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
import org.junit.Ignore;
@@ -121,7 +120,7 @@ public class NumberDictionaryForestTest {
TrieDictionaryForest<String> dict = b.build();
dict.dump(System.out);
- NumberDictionaryBuilder<String> b2 = new NumberDictionaryBuilder<>(new StringBytesConverter());
+ NumberDictionaryBuilder<String> b2 = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
for (String str : testData)
b2.addValue(str);
NumberDictionary<String> dict2 = b2.build(0);
@@ -148,22 +147,20 @@ public class NumberDictionaryForestTest {
assertTrue(dict1.getSizeOfId() == dict2.getSizeOfId());
assertTrue(dict1.getSizeOfValue() == dict2.getSizeOfValue());
-
+
byte[] buf = new byte[dict1.getSizeOfValue()];
-
+
{
- int len = dict1.getValueBytesFromId(0, buf, 0);
- int newId = dict2.getIdFromValueBytes(buf, 0, len);
+ int newId = dict2.getIdFromValue(dict1.getValueFromId(0));
assertTrue(newId == 0);
}
{
- int len = dict1.getValueBytesFromId(1, buf, 0);
- int newId = dict2.getIdFromValueBytes(buf, 0, len);
+
+ int newId = dict2.getIdFromValue(dict1.getValueFromId(1));
assertTrue(newId == 2);
}
{
- int len = dict1.getValueBytesFromId(2, buf, 0);
- int newId = dict2.getIdFromValueBytes(buf, 0, len);
+ int newId = dict2.getIdFromValue(dict1.getValueFromId(2));
assertTrue(newId == 4);
}
}
[2/2] kylin git commit: KYLIN-2400 code review
Posted by li...@apache.org.
KYLIN-2400 code review
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/576d2dd3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/576d2dd3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/576d2dd3
Branch: refs/heads/master
Commit: 576d2dd352b11f428db1d6308e35350a2fca122e
Parents: e769bfa
Author: Li Yang <li...@apache.org>
Authored: Tue Jan 17 11:06:26 2017 +0800
Committer: Li Yang <li...@apache.org>
Committed: Tue Jan 17 11:06:26 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/Dictionary.java | 42 +++-----------------
.../apache/kylin/dict/AppendTrieDictionary.java | 39 +++++++-----------
.../apache/kylin/dict/DictionaryGenerator.java | 2 +-
.../kylin/dict/NumberDictionaryBuilder.java | 10 ++---
.../dict/NumberDictionaryForestBuilder.java | 7 +++-
.../apache/kylin/dict/TrieDictionaryForest.java | 2 +-
.../apache/kylin/dict/NumberDictionaryTest.java | 4 +-
.../kylin/dict/TimeStrDictionaryTest.java | 8 ++--
.../kylin/dict/TrieDictionaryForestTest.java | 2 +-
.../kylin/measure/raw/RawMeasureType.java | 3 ++
.../storage/gtrecord/DictGridTableTest.java | 3 +-
.../mr/steps/NumberDictionaryForestTest.java | 2 +-
12 files changed, 43 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
index 9d675f9..4b5c1c4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/Dictionary.java
@@ -94,13 +94,12 @@ abstract public class Dictionary<T> implements Serializable {
final public int getIdFromValue(T value, int roundingFlag) throws IllegalArgumentException {
if (isNullObjectForm(value))
return nullId();
- else {
- int id = getIdFromValueImpl(value, roundingFlag);
- if(id == -1){
- throw new IllegalArgumentException("Value : " + value + "no exists");
- }
- return id;
+
+ int id = getIdFromValueImpl(value, roundingFlag);
+ if (id == -1) {
+ throw new IllegalArgumentException("Value : " + value + " not exists");
}
+ return id;
}
final public boolean containsValue(T value) throws IllegalArgumentException {
@@ -137,37 +136,6 @@ abstract public class Dictionary<T> implements Serializable {
abstract protected T getValueFromIdImpl(int id);
- /**
- * Convenient form of
- * <code>getIdFromValueBytes(value, offset, len, 0)</code>
- */
-
-
- /**
- * A lower level API, return ID integer from raw value bytes. In case of not found
- * <p>
- * - if roundingFlag=0, throw IllegalArgumentException; <br>
- * - if roundingFlag<0, the closest smaller ID integer if exist; <br>
- * - if roundingFlag>0, the closest bigger ID integer if exist. <br>
- * <p>
- * Bypassing the cache layer, this could be significantly slower than getIdFromValue(T value).
- *
- * @throws IllegalArgumentException
- * if value is not found in dictionary and rounding is off;
- * or if rounding cannot find a smaller or bigger ID
- */
-
- /**
- * A lower level API, get byte values from ID, return the number of bytes
- * written. Bypassing the cache layer, this could be significantly slower
- * than getIdFromValue(T value).
- *
- * @return size of value bytes, 0 if empty string, -1 if null
- *
- * @throws IllegalArgumentException
- * if ID is not found in dictionary
- */
-
abstract public void dump(PrintStream out);
public int nullId() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 3aef967..faffcc0 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -91,14 +91,11 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
volatile private TreeMap<DictSliceKey, DictSlice> dictSliceMap;
-
// Constructor both for build and deserialize
public AppendTrieDictionary() {
enableCache();
}
-
-
public void initParams(String baseDir, int baseId, int maxId, int maxValueLength, int nValues, BytesConverter bytesConverter) throws IOException {
this.baseDir = baseDir;
this.baseId = baseId;
@@ -111,8 +108,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
public void initDictSliceMap(CachedTreeMap dictMap) throws IOException {
int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
- CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir)
- .immutable(true).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
+ CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir).immutable(true).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
newDictSliceMap.loadEntry(dictMap);
this.dictSliceMap = newDictSliceMap;
}
@@ -210,7 +206,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
this.sizeChildOffset = headIn.read();
this.sizeOfId = headIn.read();
- this.childOffsetMask = ~(((long)(BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE)) << ((sizeChildOffset - 1) * 8));
+ this.childOffsetMask = ~(((long) (BIT_IS_LAST_CHILD | BIT_IS_END_OF_VALUE)) << ((sizeChildOffset - 1) * 8));
this.firstByteOffset = sizeChildOffset + 1; // the offset from begin of node to its first value byte
} catch (Exception e) {
if (e instanceof RuntimeException)
@@ -229,7 +225,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
if (checkFlag(nodeOffset, BIT_IS_END_OF_VALUE)) {
break;
}
- nodeOffset = headSize + (int)(BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
+ nodeOffset = headSize + (int) (BytesUtil.readLong(trieBytes, nodeOffset, sizeChildOffset) & childOffsetMask);
if (nodeOffset == headSize) {
break;
}
@@ -271,7 +267,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
// find a child to continue
- int c = headSize + (int)(BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int c = headSize + (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
if (c == headSize) // has no children
return -1;
byte inpByte = inp[o];
@@ -310,7 +306,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
DictNode root = null;
while (true) {
int p = n + firstByteOffset;
- int childOffset = (int)(BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ int childOffset = (int) (BytesUtil.readLong(trieBytes, n, sizeChildOffset) & childOffsetMask);
int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
@@ -357,13 +353,13 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
lastChild = false;
}
int p = offset + firstByteOffset;
- int childOffset = (int)(BytesUtil.readLong(trieBytes, offset, sizeChildOffset) & childOffsetMask);
+ int childOffset = (int) (BytesUtil.readLong(trieBytes, offset, sizeChildOffset) & childOffsetMask);
int parLen = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
boolean isEndOfValue = checkFlag(offset, BIT_IS_END_OF_VALUE);
// Copy value overflow, the data is corrupted
if (trieBytes.length < p + parLen) {
- return false;
+ return false;
}
// Check id is fine
@@ -401,7 +397,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
throw new IllegalArgumentException("Wrong file type (magic does not match)");
DataInputStream headIn = new DataInputStream(//
- new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
+ new ByteArrayInputStream(headPartial, HEAD_SIZE_I, headPartial.length - HEAD_SIZE_I));
int headSize = headIn.readShort();
int bodyLen = headIn.readInt();
headIn.close();
@@ -665,8 +661,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
// nValueBytes
if (n.part.length > 255)
- throw new RuntimeException("Value length is " + n.part.length
- + " and larger than 255: " + Bytes.toStringBinary(n.part));
+ throw new RuntimeException("Value length is " + n.part.length + " and larger than 255: " + Bytes.toStringBinary(n.part));
BytesUtil.writeUnsigned(n.part.length, trieBytes, o, 1);
o++;
@@ -785,7 +780,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
int t = s.mbpn_sizeValueTotal + s.mbpn_nNodes * (s.mbpn_sizeNoValueBytes + s.mbpn_sizeChildOffset - 1);
// *4 because 2 MSB of offset is used for isEndOfValue & isEndChild flag
// expand t to long before *4, avoiding exceed Integer.MAX_VALUE
- if (BytesUtil.sizeForValue((long)t * 4) <= s.mbpn_sizeChildOffset - 1) {
+ if (BytesUtil.sizeForValue((long) t * 4) <= s.mbpn_sizeChildOffset - 1) {
s.mbpn_sizeChildOffset--;
s.mbpn_footprint = t;
} else
@@ -886,8 +881,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
builder = new Builder<>(resourcePath, null, dictDir, 0, 0, 0, new StringBytesConverter(), null);
} else {
logger.info("GlobalDict {} exist, append value", resourcePath);
- builder = new Builder<>(resourcePath, dictToUse, dictToUse.baseDir, dictToUse.maxId, dictToUse.maxValueLength,
- dictToUse.nValues, dictToUse.bytesConvert, dictToUse.writeDictMap());
+ builder = new Builder<>(resourcePath, dictToUse, dictToUse.baseDir, dictToUse.maxId, dictToUse.maxValueLength, dictToUse.nValues, dictToUse.bytesConvert, dictToUse.writeDictMap());
}
return builder;
@@ -926,8 +920,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
// create a new cached map with baseDir
- mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir)
- .maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).immutable(false).build();
+ mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).immutable(false).build();
if (dictMapBytes != null) {
((Writable) mutableDictSliceMap).readFields(new DataInputStream(new ByteArrayInputStream(dictMapBytes)));
}
@@ -1105,7 +1098,7 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
public synchronized AppendTrieDictionary<T> build(int baseId) throws IOException {
boolean keepAppend = releaseInstance(resourcePath);
- CachedTreeMap dictSliceMap = (CachedTreeMap)mutableDictSliceMap;
+ CachedTreeMap dictSliceMap = (CachedTreeMap) mutableDictSliceMap;
dict.initParams(baseDir, baseId, maxId, maxValueLength, nValues, bytesConverter);
dict.flushIndex(dictSliceMap, keepAppend);
dict.initDictSliceMap(dictSliceMap);
@@ -1155,7 +1148,6 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
return maxValueLength;
}
-
public void flushIndex(CachedTreeMap dictSliceMap, boolean keepAppend) throws IOException {
try (FSDataOutputStream indexOut = dictSliceMap.openIndexOutput()) {
indexOut.writeInt(baseId);
@@ -1220,14 +1212,12 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
initParams(baseDir, baseId, maxId, maxValueLength, nValues, converter);
// Create instance for deserialize data, and update to map in dict
- CachedTreeMap dictMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder()
- .baseDir(baseDir).immutable(true).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
+ CachedTreeMap dictMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().baseDir(baseDir).immutable(true).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
dictMap.readFields(input);
initDictSliceMap(dictMap);
}
}
-
@Override
public void dump(PrintStream out) {
out.println("Total " + nValues + " values, " + (dictSliceMap == null ? 0 : dictSliceMap.size()) + " slice");
@@ -1261,4 +1251,3 @@ public class AppendTrieDictionary<T> extends CacheDictionary<T> {
}
}
-
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index c23e7b5..61a0664 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -227,7 +227,7 @@ public class DictionaryGenerator {
@Override
public void init(DictionaryInfo info, int baseId) throws IOException {
this.baseId = baseId;
- this.builder = new NumberDictionaryBuilder(new NumberDictionaryForestBuilder.Number2BytesConverter());
+ this.builder = new NumberDictionaryBuilder();
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
index b9e94414..288e38f 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryBuilder.java
@@ -25,17 +25,17 @@ package org.apache.kylin.dict;
* @author yangli9
*/
@Deprecated
-public class NumberDictionaryBuilder<T> extends TrieDictionaryBuilder<T> {
+public class NumberDictionaryBuilder extends TrieDictionaryBuilder<String> {
- public NumberDictionaryBuilder(BytesConverter<T> bytesConverter) {
- super(bytesConverter);
+ public NumberDictionaryBuilder() {
+ super(new NumberDictionaryForestBuilder.Number2BytesConverter());
}
- public NumberDictionary<T> build(int baseId) {
+ public NumberDictionary build(int baseId) {
byte[] trieBytes = buildTrieBytes(baseId);
- NumberDictionary2<T> r = new NumberDictionary2<T>(trieBytes);
+ NumberDictionary2 r = new NumberDictionary2(trieBytes);
return r;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
index 7c750f4..b072599 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/NumberDictionaryForestBuilder.java
@@ -18,6 +18,8 @@
package org.apache.kylin.dict;
+import java.io.Serializable;
+
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.dict.NumberDictionary.NumberBytesCodec;
@@ -26,8 +28,9 @@ import org.apache.kylin.dict.NumberDictionary.NumberBytesCodec;
*/
public class NumberDictionaryForestBuilder extends TrieDictionaryForestBuilder<String> {
- public static class Number2BytesConverter implements BytesConverter<String>, java.io.Serializable {
-
+ public static class Number2BytesConverter implements BytesConverter<String>, Serializable {
+ private static final long serialVersionUID = 1L;
+
static final int MAX_DIGITS_BEFORE_DECIMAL_POINT = NumberDictionary.MAX_DIGITS_BEFORE_DECIMAL_POINT;
static final transient ThreadLocal<NumberBytesCodec> LOCAL = new ThreadLocal<NumberBytesCodec>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index d1b5a99..04292d2 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -129,7 +129,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
TrieDictionary<T> tree = trees.get(index);
int id = tree.getIdFromValueBytesWithoutCache(value, offset, len, roundingFlag);
- if(id == -1)
+ if (id == -1)
throw new IllegalArgumentException("Value '" + Bytes.toString(value, offset, len) + "' (" + Bytes.toStringBinary(value, offset, len) + ") not exists!");
id = id + accuOffset.get(index);
id += baseId;
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
index 38f2648..1c04745 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/NumberDictionaryTest.java
@@ -59,7 +59,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
@Test
public void testMinMax() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter());
+ NumberDictionaryBuilder builder = new NumberDictionaryBuilder();
builder.addValue("" + Long.MAX_VALUE);
builder.addValue("" + Long.MIN_VALUE);
NumberDictionary<String> dict = builder.build(0);
@@ -123,7 +123,7 @@ public class NumberDictionaryTest extends LocalFileMetadataTestCase {
int n = 100;
Set<BigDecimal> set = Sets.newHashSet();
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<String>(new NumberDictionaryForestBuilder.Number2BytesConverter());
+ NumberDictionaryBuilder builder = new NumberDictionaryBuilder();
for (int i = 0; i < n; i++) {
String num = randNumber();
if (set.add(new BigDecimal(num))) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
index f981942..e852d26 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TimeStrDictionaryTest.java
@@ -18,13 +18,13 @@
package org.apache.kylin.dict;
+import static org.junit.Assert.fail;
+
import org.apache.kylin.common.util.DateFormat;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.fail;
-
/**
*/
public class TimeStrDictionaryTest {
@@ -64,10 +64,10 @@ public class TimeStrDictionaryTest {
@Test
public void testIllegal() {
- try{
+ try {
dict.getIdFromValue("2038-01-19 03:14:07");
fail("should throw exception");
- }catch (IllegalArgumentException e){
+ } catch (IllegalArgumentException e) {
//correct
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index c7fb9c4..dd1f951 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -641,7 +641,7 @@ public class TrieDictionaryForestTest {
System.out.println("max memory:" + Runtime.getRuntime().maxMemory());
System.gc();
Thread.currentThread().sleep(1000);
- NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
+ NumberDictionaryBuilder b = new NumberDictionaryBuilder();
int k = 0;
while (true) {
b.addValue(k + "");
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
index 3b6e659..31f35e9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/raw/RawMeasureType.java
@@ -46,6 +46,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class RawMeasureType extends MeasureType<List<ByteArray>> {
+ private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(RawMeasureType.class);
@@ -103,6 +104,8 @@ public class RawMeasureType extends MeasureType<List<ByteArray>> {
@Override
public MeasureIngester<List<ByteArray>> newIngester() {
return new MeasureIngester<List<ByteArray>>() {
+ private static final long serialVersionUID = 1L;
+
//encode measure value to dictionary
@Override
public List<ByteArray> valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
index ae186e2..7500b00 100644
--- a/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
+++ b/core-storage/src/test/java/org/apache/kylin/storage/gtrecord/DictGridTableTest.java
@@ -35,7 +35,6 @@ import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.cube.gridtable.CubeCodeSystem;
import org.apache.kylin.dict.NumberDictionaryBuilder;
-import org.apache.kylin.dict.NumberDictionaryForestBuilder;
import org.apache.kylin.dict.StringBytesConverter;
import org.apache.kylin.dict.TrieDictionaryBuilder;
import org.apache.kylin.dimension.DictionaryDimEnc;
@@ -602,7 +601,7 @@ public class DictGridTableTest extends LocalFileMetadataTestCase {
@SuppressWarnings("rawtypes")
private static Dictionary newDictionaryOfInteger() {
- NumberDictionaryBuilder<String> builder = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
+ NumberDictionaryBuilder builder = new NumberDictionaryBuilder();
builder.addValue("10");
builder.addValue("20");
builder.addValue("30");
http://git-wip-us.apache.org/repos/asf/kylin/blob/576d2dd3/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index 72793f5..1908992 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -120,7 +120,7 @@ public class NumberDictionaryForestTest {
TrieDictionaryForest<String> dict = b.build();
dict.dump(System.out);
- NumberDictionaryBuilder<String> b2 = new NumberDictionaryBuilder<>(new NumberDictionaryForestBuilder.Number2BytesConverter());
+ NumberDictionaryBuilder b2 = new NumberDictionaryBuilder();
for (String str : testData)
b2.addValue(str);
NumberDictionary<String> dict2 = b2.build(0);