You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/07/08 10:22:42 UTC
[2/3] kylin git commit: KYLIN-1843: support LRU cache on
SnapshotManager
KYLIN-1843: support LRU cache on SnapshotManager
Signed-off-by: Li Yang <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b45c945
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b45c945
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b45c945
Branch: refs/heads/master
Commit: 6b45c945af9948c17a971d530c0153b3564f1985
Parents: 7dc8b06
Author: Yiming Liu <li...@gmail.com>
Authored: Thu Jul 7 12:30:25 2016 +0800
Committer: Li Yang <li...@apache.org>
Committed: Fri Jul 8 18:13:55 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 4 ++
.../kylin/dict/lookup/SnapshotManager.java | 42 +++++++++++++++-----
2 files changed, 37 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b45c945/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index bfbaaa4..6b528f7 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -502,6 +502,10 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.dict.cache.max.entry", "3000"));
}
+ public int getCachedSnapshotMaxEntrySize(){
+ return Integer.parseInt(getOptional("kylin.snapshot.cache.max.entry", "500"));
+ }
+
public boolean getQueryRunLocalCoprocessor() {
return Boolean.parseBoolean(getOptional("kylin.query.run.local.coprocessor", "false"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b45c945/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
index 6e93ae4..c7b0d26 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/SnapshotManager.java
@@ -21,6 +21,8 @@ package org.apache.kylin.dict.lookup;
import java.io.IOException;
import java.util.NavigableSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -31,6 +33,12 @@ import org.apache.kylin.source.ReadableTable.TableSignature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
/**
* @author yangli9
*/
@@ -61,33 +69,49 @@ public class SnapshotManager {
// ============================================================================
private KylinConfig config;
- private ConcurrentHashMap<String, SnapshotTable> snapshotCache; // resource
+ private LoadingCache<String, SnapshotTable> snapshotCache; // resource
// path ==>
// SnapshotTable
private SnapshotManager(KylinConfig config) {
this.config = config;
- snapshotCache = new ConcurrentHashMap<String, SnapshotTable>();
+ this.snapshotCache = CacheBuilder.newBuilder().removalListener(new RemovalListener<String, SnapshotTable>() {
+ @Override
+ public void onRemoval(RemovalNotification<String, SnapshotTable> notification) {
+ SnapshotManager.logger.info("Snapshot with resource path " + notification.getKey() + " is removed due to " + notification.getCause());
+ }
+ }).maximumSize(config.getCachedSnapshotMaxEntrySize())//
+ .expireAfterWrite(1, TimeUnit.DAYS).build(new CacheLoader<String, SnapshotTable>() {
+ @Override
+ public SnapshotTable load(String key) throws Exception {
+ SnapshotTable snapshotTable = SnapshotManager.this.load(key, true);
+ return snapshotTable;
+ }
+ });
}
public void wipeoutCache() {
- snapshotCache.clear();
+ snapshotCache.invalidateAll();
}
public SnapshotTable getSnapshotTable(String resourcePath) throws IOException {
- SnapshotTable r = snapshotCache.get(resourcePath);
- if (r == null) {
- r = load(resourcePath, true);
- snapshotCache.put(resourcePath, r);
+ try {
+ SnapshotTable r = snapshotCache.get(resourcePath);
+ if (r == null) {
+ r = load(resourcePath, true);
+ snapshotCache.put(resourcePath, r);
+ }
+ return r;
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e.getCause());
}
- return r;
}
public void removeSnapshot(String resourcePath) throws IOException {
ResourceStore store = MetadataManager.getInstance(this.config).getStore();
store.deleteResource(resourcePath);
- snapshotCache.remove(resourcePath);
+ snapshotCache.invalidate(resourcePath);
}
public SnapshotTable buildSnapshot(ReadableTable table, TableDesc tableDesc) throws IOException {