You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/04/26 06:26:48 UTC
[06/50] incubator-kylin git commit: KYLIN-680, done
KYLIN-680, done
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ec46a943
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ec46a943
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ec46a943
Branch: refs/heads/streaming-localdict
Commit: ec46a943a22833b241b98abd75c7738f5f8d58a2
Parents: 5f57a71
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Apr 14 13:06:08 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Apr 14 13:06:08 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/dict/DateStrDictionary.java | 8 +-
.../java/org/apache/kylin/dict/Dictionary.java | 9 ++
.../org/apache/kylin/dict/NumberDictionary.java | 39 +++++
.../org/apache/kylin/dict/TrieDictionary.java | 143 +++++++++++++++----
4 files changed, 167 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 4523e67..e0231c5 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -125,7 +125,7 @@ public class DateStrDictionary extends Dictionary<String> {
}
@Override
- final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+ final protected byte[] getValueBytesFromIdImpl(int id) {
String date = getValueFromId(id);
byte bytes[];
try {
@@ -133,6 +133,12 @@ public class DateStrDictionary extends Dictionary<String> {
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e); // never happen
}
+ return bytes;
+ }
+
+ @Override
+ final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+ byte bytes[] = getValueBytesFromIdImpl(id);
System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
return bytes.length;
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index 815b06d..d6addb5 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -144,6 +144,15 @@ abstract public class Dictionary<T> implements Writable {
abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
+ final public byte[] getValueBytesFromId(int id) {
+ if (isNullId(id))
+ return BytesUtil.EMPTY_BYTE_ARRAY;
+ else
+ return getValueBytesFromIdImpl(id);
+ }
+
+ abstract protected byte[] getValueBytesFromIdImpl(int id);
+
/**
* A lower level API, get byte values from ID, return the number of bytes
* written. Bypassing the cache layer, this could be significantly slower
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index f200263..787d50c 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -185,4 +185,43 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
return codec.decodeNumber(returnValue, offset);
}
+ @Override
+ public void enableIdToValueBytesCache() {
+ enableIdToValueBytesCache(new EnableIdToValueBytesCacheVisitor() {
+ NumberBytesCodec codec = getCodec();
+ byte[] tmp = new byte[getSizeOfValue()];
+
+ @Override
+ public byte[] getBuffer() {
+ return codec.buf;
+ }
+
+ @Override
+ public byte[] makeValueBytes(byte[] buf, int length) {
+ // the given buf is the codec buf, which we returned in getBuffer()
+ codec.bufOffset = 0;
+ codec.bufLen = length;
+ int numLen = codec.decodeNumber(tmp, 0);
+
+ byte[] result = new byte[numLen];
+ System.arraycopy(tmp, 0, result, 0, numLen);
+ return result;
+ }
+ });
+ }
+
+
+ public static void main(String[] args) throws Exception {
+ NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<String>(new StringBytesConverter());
+ b.addValue("10");
+ b.addValue("100");
+ b.addValue("40");
+ b.addValue("7");
+ TrieDictionary<String> dict = b.build(0);
+
+ dict.enableIdToValueBytesCache();
+ for (int i = 0; i <= dict.getMaxId(); i++) {
+ System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index bf40eac..b66f02d 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -28,6 +28,7 @@ import java.lang.ref.SoftReference;
import java.util.Arrays;
import java.util.HashMap;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kylin.common.util.BytesUtil;
import org.apache.kylin.common.util.ClassUtil;
import org.slf4j.Logger;
@@ -74,10 +75,13 @@ public class TrieDictionary<T> extends Dictionary<T> {
transient private int childOffsetMask;
transient private int firstByteOffset;
- transient private boolean enableCache = true;
+ transient private boolean enableValueCache = true;
transient private SoftReference<HashMap> valueToIdCache;
transient private SoftReference<Object[]> idToValueCache;
+ transient private boolean enableIdToValueBytesCache = false;
+ transient private byte[][] idToValueBytesCache;
+
public TrieDictionary() { // default constructor for Writable interface
}
@@ -115,7 +119,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
throw new RuntimeException(e);
}
- if (enableCache) {
+ if (enableValueCache) {
valueToIdCache = new SoftReference<HashMap>(new HashMap());
idToValueCache = new SoftReference<Object[]>(new Object[nValues]);
}
@@ -143,7 +147,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
@Override
final protected int getIdFromValueImpl(T value, int roundingFlag) {
- if (enableCache && roundingFlag == 0) {
+ if (enableValueCache && roundingFlag == 0) {
HashMap cache = valueToIdCache.get(); // SoftReference to skip cache
// gracefully when short of
// memory
@@ -255,12 +259,10 @@ public class TrieDictionary<T> extends Dictionary<T> {
@Override
final protected T getValueFromIdImpl(int id) {
- if (enableCache) {
+ if (enableValueCache) {
Object[] cache = idToValueCache.get(); // SoftReference to skip cache gracefully when short of memory
if (cache != null) {
int seq = calcSeqNoFromId(id);
- if (seq < 0 || seq >= nValues)
- throw new IllegalArgumentException("Not a valid ID: " + id);
if (cache[seq] != null)
return (T) cache[seq];
@@ -278,12 +280,27 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
@Override
- protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
- if (id < baseId || id >= baseId + nValues)
- throw new IllegalArgumentException("Not a valid ID: " + id);
+ protected byte[] getValueBytesFromIdImpl(int id) {
+ if (enableIdToValueBytesCache) {
+ int seq = calcSeqNoFromId(id);
+ return idToValueBytesCache[seq];
+ }
- int seq = calcSeqNoFromId(id);
+ byte[] buf = new byte[maxValueLength];
+ int len = getValueBytesFromIdImpl(id, buf, 0);
+ if (len == buf.length) {
+ return buf;
+ } else {
+ byte[] result = new byte[len];
+ System.arraycopy(buf, 0, result, 0, len);
+ return result;
+ }
+ }
+
+ @Override
+ protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+ int seq = calcSeqNoFromId(id);
return lookupValueFromSeqNo(headSize, seq, returnValue, offset);
}
@@ -336,6 +353,68 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
}
+ public void enableIdToValueBytesCache() {
+ enableIdToValueBytesCache(new EnableIdToValueBytesCacheVisitor() {
+ @Override
+ public byte[] getBuffer() {
+ return new byte[getSizeOfValue()];
+ }
+
+ @Override
+ public byte[] makeValueBytes(byte[] buf, int length) {
+ byte[] valueBytes = new byte[length];
+ System.arraycopy(buf, 0, valueBytes, 0, length);
+ return valueBytes;
+ }
+ });
+ }
+
+ interface EnableIdToValueBytesCacheVisitor {
+ byte[] getBuffer();
+
+ byte[] makeValueBytes(byte[] buf, int length);
+ }
+
+ protected void enableIdToValueBytesCache(EnableIdToValueBytesCacheVisitor visitor) {
+ enableIdToValueBytesCache = true;
+ idToValueBytesCache = new byte[nValues][];
+ enableIdToValueBytesCache_recursion(headSize, 0, visitor.getBuffer(), 0, visitor);
+ }
+
+ private void enableIdToValueBytesCache_recursion(int n, int seq, byte[] buf, int tail, EnableIdToValueBytesCacheVisitor visitor) {
+ // write current node value
+ int p = n + firstByteOffset;
+ int len = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
+ System.arraycopy(trieBytes, p, buf, tail, len);
+ tail += len;
+
+ // if the value is ended
+ boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
+ if (isEndOfValue) {
+ idToValueBytesCache[seq] = visitor.makeValueBytes(buf, tail);
+ seq++;
+ }
+
+ // find a child to continue
+ int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+ if (c == headSize) // has no children
+ return;
+
+ // process each child
+ while (true) {
+ enableIdToValueBytesCache_recursion(c, seq, buf, tail, visitor);
+
+ int nValuesBeneath = BytesUtil.readUnsigned(trieBytes, c + sizeChildOffset, sizeNoValuesBeneath);
+ seq += nValuesBeneath;
+
+ // go next child
+ if (checkFlag(c, BIT_IS_LAST_CHILD))
+ break; // no more child? we are done
+ p = c + firstByteOffset;
+ c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1);
+ }
+ }
+
private boolean checkFlag(int offset, int bit) {
return (trieBytes[offset] & bit) > 0;
}
@@ -348,7 +427,11 @@ public class TrieDictionary<T> extends Dictionary<T> {
}
private int calcSeqNoFromId(int id) {
- return id - baseId;
+ int seq = id - baseId;
+ if (seq < 0 || seq >= nValues) {
+ throw new IllegalArgumentException("Not a valid ID: " + id);
+ }
+ return seq;
}
@Override
@@ -404,27 +487,25 @@ public class TrieDictionary<T> extends Dictionary<T> {
public static void main(String[] args) throws Exception {
TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
- // b.addValue("part");
- // b.print();
- // b.addValue("part");
- // b.print();
- // b.addValue("par");
- // b.print();
- // b.addValue("partition");
- // b.print();
- // b.addValue("party");
- // b.print();
- // b.addValue("parties");
- // b.print();
- // b.addValue("paint");
- // b.print();
- b.addValue("-000000.41");
- b.addValue("0000101.81");
- b.addValue("6779331");
- String t = "0000001.6131";
+ b.addValue("part");
+ b.print();
+ b.addValue("part");
+ b.print();
+ b.addValue("par");
+ b.print();
+ b.addValue("partition");
+ b.print();
+ b.addValue("party");
+ b.print();
+ b.addValue("parties");
+ b.print();
+ b.addValue("paint");
+ b.print();
TrieDictionary<String> dict = b.build(0);
- System.out.println(dict.getIdFromValue(t, -1));
- System.out.println(dict.getIdFromValue(t, 1));
+ dict.enableIdToValueBytesCache();
+ for (int i = 0; i <= dict.getMaxId(); i++) {
+ System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
+ }
}
}