You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/26 14:26:43 UTC
[3/5] kylin git commit: KYLIN-2322 make TrieDictionaryForest
configurable, off by default
KYLIN-2322 make TrieDictionaryForest configurable, off by default
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/440d415a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/440d415a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/440d415a
Branch: refs/heads/yang22-hbase1.x
Commit: 440d415a522e0ecb5e063b037a8ae168e74bd614
Parents: cf1aa6c
Author: Yang Li <li...@apache.org>
Authored: Mon Dec 26 21:12:20 2016 +0800
Committer: Yang Li <li...@apache.org>
Committed: Mon Dec 26 21:12:20 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ++
.../apache/kylin/dict/DictionaryGenerator.java | 64 ++++++++++++++++++--
2 files changed, 63 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/440d415a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index ceda95c..194f158 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -216,6 +216,10 @@ abstract public class KylinConfigBase implements Serializable {
// DICTIONARY & SNAPSHOT
// ============================================================================
+ public boolean isUseForestTrieDictionary() {
+ return Boolean.parseBoolean(getOptional("kylin.dictionary.use-forest-trie", "false"));
+ }
+
public int getTrieDictionaryForestMaxTrieSizeMB() {
return Integer.parseInt(getOptional("kylin.dictionary.forest-trie-max-mb", "500"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/440d415a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index cd13d59..6f4f2c4 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.common.util.Dictionary;
import org.apache.kylin.metadata.datatype.DataType;
@@ -49,10 +50,12 @@ public class DictionaryGenerator {
builder = new DateDictBuilder();
else
builder = new TimeDictBuilder();
- } else if (dataType.isNumberFamily()) {
- builder = new NumberDictBuilder();
} else {
- builder = new StringDictBuilder();
+ boolean useForest = KylinConfig.getInstanceFromEnv().isUseForestTrieDictionary();
+ if (dataType.isNumberFamily())
+ builder = useForest ? new NumberTrieDictForestBuilder() : new NumberTrieDictBuilder();
+ else
+ builder = useForest ? new StringTrieDictForestBuilder() : new StringTrieDictBuilder();
}
return builder;
}
@@ -168,7 +171,32 @@ public class DictionaryGenerator {
}
}
- private static class StringDictBuilder implements IDictionaryBuilder {
+ private static class StringTrieDictBuilder implements IDictionaryBuilder {
+ int baseId;
+ TrieDictionaryBuilder builder;
+
+ @Override
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ this.baseId = baseId;
+ this.builder = new TrieDictionaryBuilder(new StringBytesConverter());
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (value == null)
+ return false;
+
+ builder.addValue(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
+ return builder.build(baseId);
+ }
+ }
+
+ private static class StringTrieDictForestBuilder implements IDictionaryBuilder {
TrieDictionaryForestBuilder builder;
@Override
@@ -191,7 +219,33 @@ public class DictionaryGenerator {
}
}
- private static class NumberDictBuilder implements IDictionaryBuilder {
+ @SuppressWarnings("deprecation")
+ private static class NumberTrieDictBuilder implements IDictionaryBuilder {
+ int baseId;
+ NumberDictionaryBuilder builder;
+
+ @Override
+ public void init(DictionaryInfo info, int baseId) throws IOException {
+ this.baseId = baseId;
+ this.builder = new NumberDictionaryBuilder(new StringBytesConverter());
+ }
+
+ @Override
+ public boolean addValue(String value) {
+ if (StringUtils.isBlank(value)) // empty string is treated as null
+ return false;
+
+ builder.addValue(value);
+ return true;
+ }
+
+ @Override
+ public Dictionary<String> build() throws IOException {
+ return builder.build(baseId);
+ }
+ }
+
+ private static class NumberTrieDictForestBuilder implements IDictionaryBuilder {
NumberDictionaryForestBuilder builder;
@Override