You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/04/26 06:26:48 UTC

[06/50] incubator-kylin git commit: KYLIN-680, done

KYLIN-680, done


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ec46a943
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ec46a943
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ec46a943

Branch: refs/heads/streaming-localdict
Commit: ec46a943a22833b241b98abd75c7738f5f8d58a2
Parents: 5f57a71
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Apr 14 13:06:08 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Apr 14 13:06:08 2015 +0800

----------------------------------------------------------------------
 .../apache/kylin/dict/DateStrDictionary.java    |   8 +-
 .../java/org/apache/kylin/dict/Dictionary.java  |   9 ++
 .../org/apache/kylin/dict/NumberDictionary.java |  39 +++++
 .../org/apache/kylin/dict/TrieDictionary.java   | 143 +++++++++++++++----
 4 files changed, 167 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
index 4523e67..e0231c5 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DateStrDictionary.java
@@ -125,7 +125,7 @@ public class DateStrDictionary extends Dictionary<String> {
     }
 
     @Override
-    final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+    final protected byte[] getValueBytesFromIdImpl(int id) {
         String date = getValueFromId(id);
         byte bytes[];
         try {
@@ -133,6 +133,12 @@ public class DateStrDictionary extends Dictionary<String> {
         } catch (UnsupportedEncodingException e) {
             throw new RuntimeException(e); // never happen
         }
+        return bytes;
+    }
+
+    @Override
+    final protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+        byte bytes[] = getValueBytesFromIdImpl(id);
         System.arraycopy(bytes, 0, returnValue, offset, bytes.length);
         return bytes.length;
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
index 815b06d..d6addb5 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/Dictionary.java
@@ -144,6 +144,15 @@ abstract public class Dictionary<T> implements Writable {
 
     abstract protected int getIdFromValueBytesImpl(byte[] value, int offset, int len, int roundingFlag);
 
+    final public byte[] getValueBytesFromId(int id) {
+        if (isNullId(id))
+            return BytesUtil.EMPTY_BYTE_ARRAY;
+        else
+            return getValueBytesFromIdImpl(id);
+    }
+    
+    abstract protected byte[] getValueBytesFromIdImpl(int id);
+    
     /**
      * A lower level API, get byte values from ID, return the number of bytes
      * written. Bypassing the cache layer, this could be significantly slower

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
index f200263..787d50c 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/NumberDictionary.java
@@ -185,4 +185,43 @@ public class NumberDictionary<T> extends TrieDictionary<T> {
         return codec.decodeNumber(returnValue, offset);
     }
 
+    @Override
+    public void enableIdToValueBytesCache() {
+        enableIdToValueBytesCache(new EnableIdToValueBytesCacheVisitor() {
+            NumberBytesCodec codec = getCodec();
+            byte[] tmp = new byte[getSizeOfValue()];
+            
+            @Override
+            public byte[] getBuffer() {
+                return codec.buf;
+            }
+
+            @Override
+            public byte[] makeValueBytes(byte[] buf, int length) {
+                // the given buf is the codec buf, which we returned in getBuffer()
+                codec.bufOffset = 0;
+                codec.bufLen = length;
+                int numLen = codec.decodeNumber(tmp, 0);
+                
+                byte[] result = new byte[numLen];
+                System.arraycopy(tmp, 0, result, 0, numLen);
+                return result;
+            }
+        });
+    }
+
+
+    public static void main(String[] args) throws Exception {
+        NumberDictionaryBuilder<String> b = new NumberDictionaryBuilder<String>(new StringBytesConverter());
+        b.addValue("10");
+        b.addValue("100");
+        b.addValue("40");
+        b.addValue("7");
+        TrieDictionary<String> dict = b.build(0);
+
+        dict.enableIdToValueBytesCache();
+        for (int i = 0; i <= dict.getMaxId(); i++) {
+            System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ec46a943/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
index bf40eac..b66f02d 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/TrieDictionary.java
@@ -28,6 +28,7 @@ import java.lang.ref.SoftReference;
 import java.util.Arrays;
 import java.util.HashMap;
 
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.kylin.common.util.BytesUtil;
 import org.apache.kylin.common.util.ClassUtil;
 import org.slf4j.Logger;
@@ -74,10 +75,13 @@ public class TrieDictionary<T> extends Dictionary<T> {
     transient private int childOffsetMask;
     transient private int firstByteOffset;
 
-    transient private boolean enableCache = true;
+    transient private boolean enableValueCache = true;
     transient private SoftReference<HashMap> valueToIdCache;
     transient private SoftReference<Object[]> idToValueCache;
 
+    transient private boolean enableIdToValueBytesCache = false;
+    transient private byte[][] idToValueBytesCache;
+
     public TrieDictionary() { // default constructor for Writable interface
     }
 
@@ -115,7 +119,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
                 throw new RuntimeException(e);
         }
 
-        if (enableCache) {
+        if (enableValueCache) {
             valueToIdCache = new SoftReference<HashMap>(new HashMap());
             idToValueCache = new SoftReference<Object[]>(new Object[nValues]);
         }
@@ -143,7 +147,7 @@ public class TrieDictionary<T> extends Dictionary<T> {
 
     @Override
     final protected int getIdFromValueImpl(T value, int roundingFlag) {
-        if (enableCache && roundingFlag == 0) {
+        if (enableValueCache && roundingFlag == 0) {
             HashMap cache = valueToIdCache.get(); // SoftReference to skip cache
                                                   // gracefully when short of
                                                   // memory
@@ -255,12 +259,10 @@ public class TrieDictionary<T> extends Dictionary<T> {
 
     @Override
     final protected T getValueFromIdImpl(int id) {
-        if (enableCache) {
+        if (enableValueCache) {
             Object[] cache = idToValueCache.get(); // SoftReference to skip cache gracefully when short of memory
             if (cache != null) {
                 int seq = calcSeqNoFromId(id);
-                if (seq < 0 || seq >= nValues)
-                    throw new IllegalArgumentException("Not a valid ID: " + id);
                 if (cache[seq] != null)
                     return (T) cache[seq];
 
@@ -278,12 +280,27 @@ public class TrieDictionary<T> extends Dictionary<T> {
     }
 
     @Override
-    protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
-        if (id < baseId || id >= baseId + nValues)
-            throw new IllegalArgumentException("Not a valid ID: " + id);
+    protected byte[] getValueBytesFromIdImpl(int id) {
+        if (enableIdToValueBytesCache) {
+            int seq = calcSeqNoFromId(id);
+            return idToValueBytesCache[seq];
+        }
 
-        int seq = calcSeqNoFromId(id);
+        byte[] buf = new byte[maxValueLength];
+        int len = getValueBytesFromIdImpl(id, buf, 0);
 
+        if (len == buf.length) {
+            return buf;
+        } else {
+            byte[] result = new byte[len];
+            System.arraycopy(buf, 0, result, 0, len);
+            return result;
+        }
+    }
+
+    @Override
+    protected int getValueBytesFromIdImpl(int id, byte[] returnValue, int offset) {
+        int seq = calcSeqNoFromId(id);
         return lookupValueFromSeqNo(headSize, seq, returnValue, offset);
     }
 
@@ -336,6 +353,68 @@ public class TrieDictionary<T> extends Dictionary<T> {
         }
     }
 
+    public void enableIdToValueBytesCache() {
+        enableIdToValueBytesCache(new EnableIdToValueBytesCacheVisitor() {
+            @Override
+            public byte[] getBuffer() {
+                return new byte[getSizeOfValue()];
+            }
+
+            @Override
+            public byte[] makeValueBytes(byte[] buf, int length) {
+                byte[] valueBytes = new byte[length];
+                System.arraycopy(buf, 0, valueBytes, 0, length);
+                return valueBytes;
+            }
+        });
+    }
+
+    interface EnableIdToValueBytesCacheVisitor {
+        byte[] getBuffer();
+
+        byte[] makeValueBytes(byte[] buf, int length);
+    }
+
+    protected void enableIdToValueBytesCache(EnableIdToValueBytesCacheVisitor visitor) {
+        enableIdToValueBytesCache = true;
+        idToValueBytesCache = new byte[nValues][];
+        enableIdToValueBytesCache_recursion(headSize, 0, visitor.getBuffer(), 0, visitor);
+    }
+
+    private void enableIdToValueBytesCache_recursion(int n, int seq, byte[] buf, int tail, EnableIdToValueBytesCacheVisitor visitor) {
+        // write current node value
+        int p = n + firstByteOffset;
+        int len = BytesUtil.readUnsigned(trieBytes, p - 1, 1);
+        System.arraycopy(trieBytes, p, buf, tail, len);
+        tail += len;
+
+        // if the value is ended
+        boolean isEndOfValue = checkFlag(n, BIT_IS_END_OF_VALUE);
+        if (isEndOfValue) {
+            idToValueBytesCache[seq] = visitor.makeValueBytes(buf, tail);
+            seq++;
+        }
+
+        // find a child to continue
+        int c = headSize + (BytesUtil.readUnsigned(trieBytes, n, sizeChildOffset) & childOffsetMask);
+        if (c == headSize) // has no children 
+            return;
+
+        // process each child
+        while (true) {
+            enableIdToValueBytesCache_recursion(c, seq, buf, tail, visitor);
+
+            int nValuesBeneath = BytesUtil.readUnsigned(trieBytes, c + sizeChildOffset, sizeNoValuesBeneath);
+            seq += nValuesBeneath;
+
+            // go next child
+            if (checkFlag(c, BIT_IS_LAST_CHILD))
+                break; // no more child? we are done
+            p = c + firstByteOffset;
+            c = p + BytesUtil.readUnsigned(trieBytes, p - 1, 1);
+        }
+    }
+
     private boolean checkFlag(int offset, int bit) {
         return (trieBytes[offset] & bit) > 0;
     }
@@ -348,7 +427,11 @@ public class TrieDictionary<T> extends Dictionary<T> {
     }
 
     private int calcSeqNoFromId(int id) {
-        return id - baseId;
+        int seq = id - baseId;
+        if (seq < 0 || seq >= nValues) {
+            throw new IllegalArgumentException("Not a valid ID: " + id);
+        }
+        return seq;
     }
 
     @Override
@@ -404,27 +487,25 @@ public class TrieDictionary<T> extends Dictionary<T> {
 
     public static void main(String[] args) throws Exception {
         TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter());
-        // b.addValue("part");
-        // b.print();
-        // b.addValue("part");
-        // b.print();
-        // b.addValue("par");
-        // b.print();
-        // b.addValue("partition");
-        // b.print();
-        // b.addValue("party");
-        // b.print();
-        // b.addValue("parties");
-        // b.print();
-        // b.addValue("paint");
-        // b.print();
-        b.addValue("-000000.41");
-        b.addValue("0000101.81");
-        b.addValue("6779331");
-        String t = "0000001.6131";
+        b.addValue("part");
+        b.print();
+        b.addValue("part");
+        b.print();
+        b.addValue("par");
+        b.print();
+        b.addValue("partition");
+        b.print();
+        b.addValue("party");
+        b.print();
+        b.addValue("parties");
+        b.print();
+        b.addValue("paint");
+        b.print();
         TrieDictionary<String> dict = b.build(0);
 
-        System.out.println(dict.getIdFromValue(t, -1));
-        System.out.println(dict.getIdFromValue(t, 1));
+        dict.enableIdToValueBytesCache();
+        for (int i = 0; i <= dict.getMaxId(); i++) {
+            System.out.println(Bytes.toString(dict.getValueBytesFromId(i)));
+        }
     }
 }