You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/08/24 07:28:22 UTC
[kylin] branch master updated: KYLIN-3071: Add config to reuse dict
to reduce dict size
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new e8e2052 KYLIN-3071: Add config to reuse dict to reduce dict size
e8e2052 is described below
commit e8e20529f42e6c9865d13664e31a9f0422e17086
Author: yanghao3 <ya...@xiaomi.com>
AuthorDate: Wed Jan 31 14:33:10 2018 +0800
KYLIN-3071: Add config to reuse dict to reduce dict size
---
.../org/apache/kylin/common/KylinConfigBase.java | 4 +++
.../org/apache/kylin/dict/DictionaryManager.java | 37 ++++++++++++----------
2 files changed, 24 insertions(+), 17 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f154eee..925488f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -394,6 +394,10 @@ abstract public class KylinConfigBase implements Serializable {
return Boolean.parseBoolean(this.getOptional("kylin.dictionary.growing-enabled", "false"));
}
+ public boolean isDictResuable() {
+ return Boolean.parseBoolean(this.getOptional("kylin.dictionary.resuable", "false"));
+ }
+
public int getAppendDictEntrySize() {
return Integer.parseInt(getOptional("kylin.dictionary.append-entry-size", "10000000"));
}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
index ff4b004..6b9e733 100755
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/DictionaryManager.java
@@ -18,15 +18,12 @@
package org.apache.kylin.dict;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.Lists;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.util.ClassUtil;
@@ -39,12 +36,14 @@ import org.apache.kylin.source.IReadableTable.TableSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.common.collect.Lists;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
public class DictionaryManager {
@@ -174,8 +173,12 @@ public class DictionaryManager {
for (String existing : existings) {
DictionaryInfo existingInfo = getDictionaryInfo(existing);
- if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject())) {
- return existing;
+ if (existingInfo != null) {
+ if ((config.isDictResuable() && existingInfo.getDictionaryObject().contains(dict))
+ || dict.equals(existingInfo.getDictionaryObject())) {
+ return existing;
+ }
+
}
}