You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/11/11 11:35:32 UTC
[kylin] 01/02: KYLIN-3678 CacheStateChecker may remove a cache file
that under building
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch 2.5.x
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2e1c2da1dea34e2cec5337e6f8a67fb5e419c3eb
Author: shaofengshi <sh...@apache.org>
AuthorDate: Fri Nov 9 21:34:11 2018 +0800
KYLIN-3678 CacheStateChecker may remove a cache file that under building
---
.../src/main/java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++
.../apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java | 6 ++++--
.../kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java | 8 ++++++++
3 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index b31f11c..0041402 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -451,6 +451,10 @@ abstract public class KylinConfigBase implements Serializable {
return Double.parseDouble(getOptional("kylin.snapshot.ext.local.cache.max-size-gb", "200"));
}
+ public long getExtTableSnapshotLocalCacheCheckVolatileRange() {
+ return Long.parseLong(getOptional("kylin.snapshot.ext.local.cache.check.volatile", "3600000"));
+ }
+
public boolean isShrunkenDictFromGlobalEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.dictionary.shrunken-from-global-enabled", "false"));
}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
index 559c435..c748e18 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCache.java
@@ -352,8 +352,10 @@ public class RocksDBLookupTableCache implements IExtLookupTableCache {
List<Pair<String, File>> toRemovedCachedSnapshots = Lists.newArrayList(FluentIterable.from(
allCachedSnapshots).filter(new Predicate<Pair<String, File>>() {
@Override
- public boolean apply(@Nullable Pair<String, File> input) {
- return !activeSnapshotSet.contains(input.getFirst());
+ public boolean apply(@Nullable Pair<String, File> input) {
+ long lastModified = input.getSecond().lastModified();
+ return !activeSnapshotSet.contains(input.getFirst()) && lastModified > 0
+ && lastModified < (System.currentTimeMillis() - config.getExtTableSnapshotLocalCacheCheckVolatileRange());
}
}));
for (Pair<String, File> toRemovedCachedSnapshot : toRemovedCachedSnapshots) {
diff --git a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
index 747fe23..579e499 100644
--- a/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
+++ b/core-dictionary/src/test/java/org/apache/kylin/dict/lookup/cache/RocksDBLookupTableCacheTest.java
@@ -153,7 +153,15 @@ public class RocksDBLookupTableCacheTest extends LocalFileMetadataTestCase {
ExtTableSnapshotInfoManager.getInstance(kylinConfig).removeSnapshot(snapshotInfo.getTableName(), snapshotInfo.getId());
cache.checkCacheState();
String cacheLocalPath = cache.getSnapshotCachePath(snapshotInfo.getTableName(), snapshotInfo.getId());
+ // won't cleanup because it is newly created in last 1 hour
+ assertTrue(new File(cacheLocalPath).exists());
+
+ // change the volatile value
+ kylinConfig.setProperty("kylin.snapshot.ext.local.cache.check.volatile", "0");
+ cache.checkCacheState();
+ // this time it should be removed.
assertFalse(new File(cacheLocalPath).exists());
+
cachedLookupTable = cache.getCachedLookupTable(tableDesc, snapshotInfo, false);
assertNull(cachedLookupTable);
}