You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/01/12 09:07:25 UTC
kylin git commit: KYLIN-1307 revisit growing dictionary
Repository: kylin
Updated Branches:
refs/heads/2.0-rc 3904a1767 -> 9f743fb78
KYLIN-1307 revisit growing dictionary
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9f743fb7
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9f743fb7
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9f743fb7
Branch: refs/heads/2.0-rc
Commit: 9f743fb7871d02f2c06e8fe21d288de1f502c939
Parents: 3904a17
Author: honma <ho...@ebay.com>
Authored: Tue Jan 12 16:06:39 2016 +0800
Committer: honma <ho...@ebay.com>
Committed: Tue Jan 12 16:06:39 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 5 +-
.../apache/kylin/dict/DictionaryManager.java | 58 +++++++++++++++-----
.../apache/kylin/rest/service/CacheService.java | 2 +
3 files changed, 51 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index a5ca690..4c640f3 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -412,7 +412,6 @@ public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", "3000"));
}
-
public boolean getQueryRunLocalCoprocessor() {
return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
}
@@ -453,6 +452,10 @@ public class KylinConfigBase implements Serializable {
return Integer.parseInt(this.getOptional("kylin.hbase.scan.cache_rows", "1024"));
}
+ public boolean isGrowingDictEnabled() {
+ return Boolean.parseBoolean(this.getOptional("kylin.dict.growing.enabled", "false"));
+ }
+
public int getHBaseScanMaxResultSize() {
return Integer.parseInt(this.getOptional("kylin.hbase.scan.max_result_size", "" + (5 * 1024 * 1024))); // 5 MB
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index b3cd634..cd12f01 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -139,26 +139,58 @@ public class DictionaryManager {
initDictInfo(newDict, newDictInfo);
- DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
- if (largestDictInfo != null) {
- largestDictInfo = getDictionaryInfo(largestDictInfo.getResourcePath());
- Dictionary<?> largestDictObject = largestDictInfo.getDictionaryObject();
- if (largestDictObject.contains(newDict)) {
- logger.info("dictionary content " + newDict + ", is contained by dictionary at " + largestDictInfo.getResourcePath());
- return largestDictInfo;
- } else if (newDict.contains(largestDictObject)) {
- logger.info("dictionary content " + newDict + " is by far the largest, save it");
- return saveNewDict(newDictInfo);
+ if (KylinConfig.getInstanceFromEnv().isGrowingDictEnabled()) {
+ DictionaryInfo largestDictInfo = findLargestDictInfo(newDictInfo);
+ if (largestDictInfo != null) {
+ largestDictInfo = getDictionaryInfo(largestDictInfo.getResourcePath());
+ Dictionary<?> largestDictObject = largestDictInfo.getDictionaryObject();
+ if (largestDictObject.contains(newDict)) {
+ logger.info("dictionary content " + newDict + ", is contained by dictionary at " + largestDictInfo.getResourcePath());
+ return largestDictInfo;
+ } else if (newDict.contains(largestDictObject)) {
+ logger.info("dictionary content " + newDict + " is by far the largest, save it");
+ return saveNewDict(newDictInfo);
+ } else {
+ logger.info("merge dict and save...");
+ return mergeDictionary(Lists.newArrayList(newDictInfo, largestDictInfo));
+ }
} else {
- logger.info("merge dict and save...");
- return mergeDictionary(Lists.newArrayList(newDictInfo, largestDictInfo));
+ logger.info("first dict of this column, save it directly");
+ return saveNewDict(newDictInfo);
}
} else {
- logger.info("first dict of this column, save it directly");
+ logger.info("Growing dict is not enabled");
+ String dupDict = checkDupByContent(newDictInfo, newDict);
+ if (dupDict != null) {
+ logger.info("Identical dictionary content, reuse existing dictionary at " + dupDict);
+ return getDictionaryInfo(dupDict);
+ }
+
return saveNewDict(newDictInfo);
}
}
+ private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException {
+ ResourceStore store = MetadataManager.getInstance(config).getStore();
+ ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
+ if (existings == null)
+ return null;
+
+ logger.info("{} existing dictionaries of the same column", existings.size());
+ if (existings.size() > 100) {
+ logger.warn("Too many dictionaries under {}, dict count: {}", dictInfo.getResourceDir(), existings.size());
+ }
+
+ for (String existing : existings) {
+ DictionaryInfo existingInfo = getDictionaryInfo(existing);
+ if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject())) {
+ return existing;
+ }
+ }
+
+ return null;
+ }
+
private void initDictInfo(Dictionary<?> newDict, DictionaryInfo newDictInfo) {
newDictInfo.setCardinality(newDict.getSize());
newDictInfo.setDictionaryObject(newDict);
http://git-wip-us.apache.org/repos/asf/kylin/blob/9f743fb7/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
index 8227be6..c8bc941 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CacheService.java
@@ -38,6 +38,7 @@ import org.apache.kylin.common.restclient.Broadcaster;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.engine.streaming.StreamingManager;
import org.apache.kylin.invertedindex.IIDescManager;
import org.apache.kylin.invertedindex.IIManager;
@@ -203,6 +204,7 @@ public class CacheService extends BasicService {
CubeDescManager.clearCache();
break;
case ALL:
+ DictionaryManager.clearCache();
MetadataManager.clearCache();
CubeDescManager.clearCache();
CubeManager.clearCache();