You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/12/20 11:27:27 UTC
[02/50] [abbrv] kylin git commit: KYLIN-2266 Reduce memory usage for
building global dict
KYLIN-2266 Reduce memory usage for building global dict
Signed-off-by: shaofengshi <sh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/65c58990
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/65c58990
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/65c58990
Branch: refs/heads/master-cdh5.7
Commit: 65c58990802c644ea35005c301ba2951a9b7e6a5
Parents: 5d6bfba
Author: kangkaisen <ka...@live.com>
Authored: Sun Dec 11 20:49:28 2016 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Mon Dec 12 10:55:32 2016 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfigBase.java | 9 ---------
.../java/org/apache/kylin/dict/AppendTrieDictionary.java | 6 ++----
.../org/apache/kylin/dict/AppendTrieDictionaryTest.java | 1 -
3 files changed, 2 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f46c185..d4272f9 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -245,15 +245,6 @@ abstract public class KylinConfigBase implements Serializable {
setProperty("kylin.dictionary.append-entry-size", String.valueOf(entrySize));
}
- public int getAppendDictCacheSize() {
- return Integer.parseInt(getOptional("kylin.dictionary.append-cache-size", "20"));
- }
-
- // for test
- public void setAppendDictCacheSize(int cacheSize) {
- setProperty("kylin.dictionary.append-cache-size", String.valueOf(cacheSize));
- }
-
public int getCachedSnapshotMaxEntrySize() {
return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 35aa9a5..32bfde6 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -115,10 +115,9 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
public void initDictSliceMap(CachedTreeMap dictMap) throws IOException {
- int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize();
int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
- CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir)
+ CachedTreeMap newDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir)
.immutable(true).maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictSlice.class).build();
newDictSliceMap.loadEntry(dictMap);
this.dictSliceMap = newDictSliceMap;
@@ -930,11 +929,10 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
this.bytesConverter = bytesConverter;
MAX_ENTRY_IN_SLICE = KylinConfig.getInstanceFromEnv().getAppendDictEntrySize();
- int cacheSize = KylinConfig.getInstanceFromEnv().getAppendDictCacheSize();
int maxVersions = KylinConfig.getInstanceFromEnv().getAppendDictMaxVersions();
long versionTTL = KylinConfig.getInstanceFromEnv().getAppendDictVersionTTL();
// create a new cached map with baseDir
- mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(cacheSize).baseDir(baseDir)
+ mutableDictSliceMap = CachedTreeMap.CachedTreeMapBuilder.newBuilder().maxSize(1).baseDir(baseDir)
.maxVersions(maxVersions).versionTTL(versionTTL).keyClazz(DictSliceKey.class).valueClazz(DictNode.class).immutable(false).build();
if (dictMapBytes != null) {
((Writable) mutableDictSliceMap).readFields(new DataInputStream(new ByteArrayInputStream(dictMapBytes)));
http://git-wip-us.apache.org/repos/asf/kylin/blob/65c58990/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
index a7e8152..0776599 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/AppendTrieDictionaryTest.java
@@ -63,7 +63,6 @@ public class AppendTrieDictionaryTest {
System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/localmeta");
KylinConfig config = KylinConfig.getInstanceFromEnv();
config.setAppendDictEntrySize(50000);
- config.setAppendDictCacheSize(3);
config.setProperty("kylin.env.hdfs-working-dir", BASE_DIR);
}