You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/06/16 04:28:00 UTC
incubator-kylin git commit: KYLIN-808, back port the fix to 0.7
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging bcc2588a3 -> 8d5484563
KYLIN-808, back port the fix to 0.7
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/8d548456
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/8d548456
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/8d548456
Branch: refs/heads/0.7-staging
Commit: 8d5484563397a4f75b1e6c2874f8001194325354
Parents: bcc2588
Author: Li, Yang <ya...@ebay.com>
Authored: Tue Jun 16 10:27:44 2015 +0800
Committer: Li, Yang <ya...@ebay.com>
Committed: Tue Jun 16 10:27:44 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/dict/DictionaryGenerator.java | 2 +-
.../org/apache/kylin/dict/DictionaryManager.java | 16 ++++++++++++++--
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d548456/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
index 13b34af..04c5aed 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryGenerator.java
@@ -85,7 +85,7 @@ public class DictionaryGenerator {
logger.info("Dictionary value samples: " + buf.toString());
logger.info("Dictionary cardinality " + info.getCardinality());
- if (values.size() > DICT_MAX_CARDINALITY)
+ if (dict instanceof TrieDictionary && values.size() > DICT_MAX_CARDINALITY)
throw new IllegalArgumentException("Too high cardinality is not suitable for dictionary -- " + info.getSourceTable() + "." + info.getSourceColumn() + " cardinality: " + values.size());
return dict;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/8d548456/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index 3a2cb8d..08e8f35 100644
--- a/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -35,13 +35,13 @@ import org.apache.kylin.dict.lookup.HiveTable;
import org.apache.kylin.dict.lookup.TableSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.HadoopUtil;
import org.apache.kylin.dict.lookup.ReadableTable;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.TblColRef;
public class DictionaryManager {
@@ -146,8 +146,20 @@ public class DictionaryManager {
return getDictionaryInfo(dupDict);
}
+ // check for cases where merging dicts are actually same
+ boolean identicalSourceDicts = true;
+ for (int i = 1; i < dicts.size(); ++i) {
+ if (!dicts.get(0).getDictionaryObject().equals(dicts.get(i).getDictionaryObject())) {
+ identicalSourceDicts = false;
+ break;
+ }
+ }
+ if (identicalSourceDicts) {
+ logger.info("Use one of the merging dictionaries directly");
+ return dicts.get(0);
+ }
+
Dictionary<?> newDict = DictionaryGenerator.mergeDictionaries(newDictInfo, dicts);
-
return trySaveNewDict(newDict, newDictInfo);
}