You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/11 10:05:19 UTC
incubator-kylin git commit: KYLIN-660 Make configurable of dictionary
cardinality cap
Repository: incubator-kylin
Updated Branches:
refs/heads/0.6 a4692dba6 -> 498b98450
KYLIN-660 Make configurable of dictionary cardinality cap
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/498b9845
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/498b9845
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/498b9845
Branch: refs/heads/0.6
Commit: 498b9845093af7bd9629a1e622bccd41f8400aaf
Parents: a4692db
Author: Li, Yang <ya...@ebay.com>
Authored: Thu Jun 11 16:04:54 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Thu Jun 11 16:04:54 2015 +0800
----------------------------------------------------------------------
.../java/com/kylinolap/common/KylinConfig.java | 24 +++++++-------------
.../com/kylinolap/dict/DictionaryGenerator.java | 3 ++-
2 files changed, 10 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/498b9845/common/src/main/java/com/kylinolap/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/kylinolap/common/KylinConfig.java b/common/src/main/java/com/kylinolap/common/KylinConfig.java
index 2112fe8..9dd64b4 100644
--- a/common/src/main/java/com/kylinolap/common/KylinConfig.java
+++ b/common/src/main/java/com/kylinolap/common/KylinConfig.java
@@ -392,16 +392,10 @@ public class KylinConfig {
return Integer.parseInt(getOptional(KYLIN_JOB_YARN_APP_REST_CHECK_INTERVAL_SECONDS, "60"));
}
- /**
- * @return
- */
public int getMaxConcurrentJobLimit() {
return Integer.parseInt(getOptional(KYLIN_JOB_CONCURRENT_MAX_LIMIT, "10"));
}
- /**
- * @return
- */
public String getTimeZone() {
return getOptional(KYLIN_REST_TIMEZONE, "PST");
}
@@ -413,20 +407,10 @@ public class KylinConfig {
return nodes.split("\\s*,\\s*");
}
- /**
- * @return
- */
public String getAdminDls() {
return getOptional("kylin.job.admin.dls", null);
}
- public int getScanThreshold() {
- return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
- }
-
- /**
- * @return
- */
public long getJobStepTimeout() {
return Long.parseLong(getOptional("kylin.job.step.timeout", String.valueOf(2 * 60 * 60)));
}
@@ -434,6 +418,14 @@ public class KylinConfig {
public String getServerMode() {
return this.getOptional("kylin.server.mode", "all");
}
+
+ public int getDictionaryMaxCardinality() {
+ return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
+ }
+
+ public int getScanThreshold() {
+ return Integer.parseInt(getOptional("kylin.query.scan.threshold", "10000000"));
+ }
public Long getQueryDurationCacheThreshold() {
return Long.parseLong(this.getOptional("kylin.query.cache.threshold.duration", String.valueOf(2000)));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/498b9845/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
index f474035..e1c2b07 100644
--- a/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/com/kylinolap/dict/DictionaryGenerator.java
@@ -29,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.kylinolap.common.KylinConfig;
import com.kylinolap.common.util.JsonUtil;
import com.kylinolap.dict.lookup.ReadableTable;
import com.kylinolap.dict.lookup.TableReader;
@@ -40,7 +41,7 @@ import com.kylinolap.metadata.model.schema.DataType;
@SuppressWarnings({ "rawtypes", "unchecked" })
public class DictionaryGenerator {
- private static final int DICT_MAX_CARDINALITY = 2000000; // 2 million
+ private static final int DICT_MAX_CARDINALITY = KylinConfig.getInstanceFromEnv().getDictionaryMaxCardinality();
private static final Logger logger = LoggerFactory.getLogger(DictionaryGenerator.class);