You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/25 12:19:04 UTC
kylin git commit: KYLIN-2249 Bug fix : build cube error when use
inmem but ok with layer
Repository: kylin
Updated Branches:
refs/heads/2.0.x 6c376b248 -> 4d52147e8
KYLIN-2249 Bug fix : build cube error when use inmem but ok with layer
Signed-off-by: Yang Li <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4d52147e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4d52147e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4d52147e
Branch: refs/heads/2.0.x
Commit: 4d52147e81e2581e39256e56680fd06ca1d4963c
Parents: 6c376b2
Author: xiefan46 <95...@qq.com>
Authored: Tue Apr 25 18:10:09 2017 +0800
Committer: Yang Li <li...@apache.org>
Committed: Tue Apr 25 20:17:58 2017 +0800
----------------------------------------------------------------------
.../kylin/dict/TrieDictionaryBuilder.java | 33 +++++++++++++-------
.../apache/kylin/dict/TrieDictionaryForest.java | 14 ++++-----
.../kylin/dict/TrieDictionaryForestBuilder.java | 2 +-
.../kylin/dict/TrieDictionaryForestTest.java | 11 +++++++
.../apache/kylin/dict/TrieDictionaryTest.java | 10 ++++++
.../kylin/dimension/DictionaryDimEnc.java | 2 +-
6 files changed, 51 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d52147e/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
index 102c49e..1750ac1 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryBuilder.java
@@ -34,11 +34,11 @@ import org.apache.kylin.common.util.BytesUtil;
* Builds a dictionary using Trie structure. All values are taken in byte[] form
* and organized in a Trie with ordering. Then numeric IDs are assigned in
* sequence.
- *
+ *
* @author yangli9
*/
public class TrieDictionaryBuilder<T> {
-
+
private static final int _2GB = 2000000000;
public static class Node {
@@ -76,6 +76,8 @@ public class TrieDictionaryBuilder<T> {
private Node root;
protected BytesConverter<T> bytesConverter;
+ private boolean hasValue = false;
+
public TrieDictionaryBuilder(BytesConverter<T> bytesConverter) {
this.root = new Node(new byte[0], false);
this.bytesConverter = bytesConverter;
@@ -91,6 +93,7 @@ public class TrieDictionaryBuilder<T> {
}
private void addValueR(Node node, byte[] value, int start) {
+ hasValue = true;
// match the value part of current node
int i = 0, j = start;
int n = node.part.length, nn = value.length;
@@ -179,10 +182,8 @@ public class TrieDictionaryBuilder<T> {
public static class Stats {
public int nValues; // number of values in total
- public int nValueBytesPlain; // number of bytes for all values
- // uncompressed
- public int nValueBytesCompressed; // number of values bytes in Trie
- // (compressed)
+ public int nValueBytesPlain; // number of bytes for all values uncompressed
+ public int nValueBytesCompressed; // number of values bytes in Trie (compressed)
public int maxValueLength; // size of longest value in bytes
// the trie is multi-byte-per-node
@@ -234,7 +235,13 @@ public class TrieDictionaryBuilder<T> {
}
}
- /** out print some statistics of the trie and the dictionary built from it */
+ public boolean isHasValue() {
+ return hasValue;
+ }
+
+ /**
+ * out print some statistics of the trie and the dictionary built from it
+ */
public Stats stats() {
// calculate nEndValueBeneath
traversePostOrder(new Visitor() {
@@ -313,7 +320,9 @@ public class TrieDictionaryBuilder<T> {
return s;
}
- /** out print trie for debug */
+ /**
+ * out print trie for debug
+ */
public void print() {
print(System.out);
}
@@ -396,11 +405,11 @@ public class TrieDictionaryBuilder<T> {
/**
* Flatten the trie into a byte array for a minimized memory footprint.
* Lookup remains fast. Cost is inflexibility to modify (becomes immutable).
- *
+ * <p>
* Flattened node structure is HEAD + NODEs, for each node:
* - o byte, offset to child node, o = stats.mbpn_sizeChildOffset
- * - 1 bit, isLastChild flag, the 1st MSB of o
- * - 1 bit, isEndOfValue flag, the 2nd MSB of o
+ * - 1 bit, isLastChild flag, the 1st MSB of o
+ * - 1 bit, isEndOfValue flag, the 2nd MSB of o
* - c byte, number of values beneath, c = stats.mbpn_sizeNoValueBeneath
* - 1 byte, number of value bytes
* - n byte, value bytes
@@ -417,7 +426,7 @@ public class TrieDictionaryBuilder<T> {
Stats stats = stats();
int sizeNoValuesBeneath = stats.mbpn_sizeNoValueBeneath;
int sizeChildOffset = stats.mbpn_sizeChildOffset;
-
+
if (stats.mbpn_footprint <= 0) // must never happen, but let us be cautious
throw new IllegalStateException("Too big dictionary, dictionary cannot be bigger than 2GB");
if (stats.mbpn_footprint > _2GB)
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d52147e/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
index 1023892..09d5bc2 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForest.java
@@ -63,7 +63,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
public TrieDictionaryForest(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, //
- ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) {
+ ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId) {
init(trees, valueDivide, accuOffset, bytesConverter, baseId);
}
@@ -342,7 +342,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
initSizeOfValue();
}
- private void initMaxValueForEachTrie(){
+ private void initMaxValueForEachTrie() {
//init max value
this.maxValue = new ArrayList<>();
if (this.trees == null || trees.isEmpty()) {
@@ -356,7 +356,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
}
}
- private void initMaxId(){
+ private void initMaxId() {
if (trees.isEmpty()) {
this.maxId = baseId - 1;
return;
@@ -365,7 +365,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
this.maxId = accuOffset.get(index) + trees.get(index).getMaxId() + baseId;
}
- private void initMinId(){
+ private void initMinId() {
if (trees.isEmpty()) {
this.minId = baseId;
return;
@@ -373,8 +373,8 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
this.minId = trees.get(0).getMinId() + baseId;
}
- private void initSizeOfId(){
- if (trees.isEmpty()){
+ private void initSizeOfId() {
+ if (trees.isEmpty()) {
this.sizeOfId = 1;
return;
}
@@ -383,7 +383,7 @@ public class TrieDictionaryForest<T> extends CacheDictionary<T> {
this.sizeOfId = BytesUtil.sizeForValue(baseId + maxOffset + lastTree.getMaxId() + 1L);
}
- private void initSizeOfValue(){
+ private void initSizeOfValue() {
int maxValue = 0;
for (TrieDictionary<T> tree : trees)
maxValue = Math.max(maxValue, tree.getSizeOfValue());
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d52147e/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
index 69da472..0e5e63e 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TrieDictionaryForestBuilder.java
@@ -105,7 +105,7 @@ public class TrieDictionaryForestBuilder<T> {
}
public TrieDictionaryForest<T> build() {
- if (curTreeSize != 0) { //last tree
+ if (trieBuilder.isHasValue()) { //last tree
TrieDictionary<T> tree = trieBuilder.build(0);
addTree(tree);
reset();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d52147e/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
index dd1f951..82380b3 100755
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryForestTest.java
@@ -133,6 +133,17 @@ public class TrieDictionaryForestTest {
}
@Test
+ public void testAllNullValue() {
+ ArrayList<String> strs = new ArrayList<String>();
+ strs.add("");
+ int maxTreeSize = 10;
+ TrieDictionaryForestBuilder<String> builder = newDictBuilder(strs, 0, maxTreeSize);
+ TrieDictionaryForest<String> dict = builder.build();
+ assertEquals(1, dict.getSize());
+ assertEquals(0, dict.getIdFromValue(""));
+ }
+
+ @Test
public void testBigDataSet() {
//h=generate data
ArrayList<String> strs = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d52147e/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
index 22a93a0..13c83ac 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/TrieDictionaryTest.java
@@ -221,6 +221,16 @@ public class TrieDictionaryTest {
testStringDictionary(str, null);
}
+ @Test
+ public void testAllNullValue() {
+ ArrayList<String> strs = new ArrayList<String>();
+ strs.add("");
+ TrieDictionaryBuilder<String> builder = newDictBuilder(strs);
+ TrieDictionary<String> dict = builder.build(0);
+ assertEquals(1, dict.getSize());
+ assertEquals(0, dict.getIdFromValue(""));
+ }
+
private static void benchmarkStringDictionary(Iterable<String> str) throws IOException {
TrieDictionaryBuilder<String> b = newDictBuilder(str);
b.stats().print();
http://git-wip-us.apache.org/repos/asf/kylin/blob/4d52147e/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
index 46f9117..b022d84 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
@@ -30,7 +30,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class DictionaryDimEnc extends DimensionEncoding implements Serializable{
+public class DictionaryDimEnc extends DimensionEncoding implements Serializable {
private static final long serialVersionUID = 1L;
private static final Logger logger = LoggerFactory.getLogger(DictionaryDimEnc.class);