You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by kt...@apache.org on 2017/10/27 17:05:57 UTC
[fluo] branch master updated: fixes #939 made transactor cache
configurable (#956)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo.git
The following commit(s) were added to refs/heads/master by this push:
new 955c86f fixes #939 made transactor cache configurable (#956)
955c86f is described below
commit 955c86fadcc207eecd2a6af464ad51f15e3a223c
Author: Kenneth McFarland <ke...@gmail.com>
AuthorDate: Tue Oct 24 11:54:49 2017 -0700
fixes #939 made transactor cache configurable (#956)
---
.../fluo/core/impl/FluoConfigurationImpl.java | 68 +++++++++++++++++-----
.../org/apache/fluo/core/impl/TransactorCache.java | 14 +++--
.../org/apache/fluo/core/impl/TxInfoCache.java | 2 +-
.../org/apache/fluo/core/impl/VisibilityCache.java | 4 +-
4 files changed, 64 insertions(+), 24 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/fluo/core/impl/FluoConfigurationImpl.java b/modules/core/src/main/java/org/apache/fluo/core/impl/FluoConfigurationImpl.java
index aad97a1..b742a50 100644
--- a/modules/core/src/main/java/org/apache/fluo/core/impl/FluoConfigurationImpl.java
+++ b/modules/core/src/main/java/org/apache/fluo/core/impl/FluoConfigurationImpl.java
@@ -104,21 +104,21 @@ public class FluoConfigurationImpl {
return m;
}
- public static final String TX_INFO_CACHE_SIZE = FLUO_IMPL_PREFIX + ".tx.failed.cache.size.mb";
- public static final long TX_INFO_CACHE_SIZE_DEFAULT = 10_000_000;
+ public static final String TX_INFO_CACHE_WEIGHT = FLUO_IMPL_PREFIX + ".tx.failed.cache.weight.mb";
+ public static final long TX_INFO_CACHE_WEIGHT_DEFAULT = 10_000_000;
/**
- * Gets the cache size
+ * Gets the txinfo cache weight
*
* @param conf The FluoConfiguration
- * @return The size of the cache value from the property value {@value #TX_INFO_CACHE_SIZE}
- * if it is set, else the value of the default value {@value #TX_INFO_CACHE_SIZE_DEFAULT}
+ * @return The size of the cache value from the property value {@value #TX_INFO_CACHE_WEIGHT}
+ * if it is set, else the value of the default value {@value #TX_INFO_CACHE_WEIGHT_DEFAULT}
*/
- public static long getTxInfoCacheSize(FluoConfiguration conf) {
- long size = conf.getLong(TX_INFO_CACHE_SIZE, TX_INFO_CACHE_SIZE_DEFAULT);
+ public static long getTxInfoCacheWeight(FluoConfiguration conf) {
+ long size = conf.getLong(TX_INFO_CACHE_WEIGHT, TX_INFO_CACHE_WEIGHT_DEFAULT);
if (size <= 0) {
- throw new IllegalArgumentException("Cache size must be positive for " + TX_INFO_CACHE_SIZE);
+ throw new IllegalArgumentException("Cache size must be positive for " + TX_INFO_CACHE_WEIGHT);
}
return size;
}
@@ -144,22 +144,23 @@ public class FluoConfigurationImpl {
return tu.convert(millis, TimeUnit.MILLISECONDS);
}
- public static final String VISIBILITY_CACHE_SIZE = FLUO_IMPL_PREFIX + ".visibility.cache.size.mb";
- public static final long VISIBILITY_CACHE_SIZE_DEFAULT = 10_000_000;
+ public static final String VISIBILITY_CACHE_WEIGHT =
+ FLUO_IMPL_PREFIX + ".visibility.cache.weight.mb";
+ public static final long VISIBILITY_CACHE_WEIGHT_DEFAULT = 10_000_000;
/**
- * Gets the cache size
+ * Gets the visibility cache weight
*
* @param conf The FluoConfiguration
- * @return The size of the cache value from the property value {@value #VISIBILITY_CACHE_SIZE}
- * if it is set, else the value of the default value {@value #VISIBILITY_CACHE_SIZE_DEFAULT}
+ * @return The size of the cache value from the property value {@value #VISIBILITY_CACHE_WEIGHT}
+ * if it is set, else the value of the default value {@value #VISIBILITY_CACHE_WEIGHT_DEFAULT}
*/
- public static long getVisibilityCacheSize(FluoConfiguration conf) {
- long size = conf.getLong(VISIBILITY_CACHE_SIZE, VISIBILITY_CACHE_SIZE_DEFAULT);
+ public static long getVisibilityCacheWeight(FluoConfiguration conf) {
+ long size = conf.getLong(VISIBILITY_CACHE_WEIGHT, VISIBILITY_CACHE_WEIGHT_DEFAULT);
if (size <= 0) {
throw new IllegalArgumentException(
- "Cache size must be positive for " + VISIBILITY_CACHE_SIZE);
+ "Cache size must be positive for " + VISIBILITY_CACHE_WEIGHT);
}
return size;
}
@@ -185,6 +186,41 @@ public class FluoConfigurationImpl {
return tu.convert(millis, TimeUnit.MILLISECONDS);
}
+ private static final String TRANSACTOR_MAX_CACHE_SIZE =
+ FLUO_IMPL_PREFIX + ".transactor.cache.max.size";
+ private static final long TRANSACTOR_MAX_CACHE_SIZE_DEFAULT = 32768; // this equals 2^15
+
+ /**
+ * Gets the specified number of entries the cache can contain, this gets the value
+ * of {@value #TRANSACTOR_MAX_CACHE_SIZE} if set, the default
+ * {@value #TRANSACTOR_CACHE_TIMEOUT_DEFAULT} otherwise
+ *
+ * @param conf The FluoConfiguartion
+ * @return The maximum number of entries permitted in this cache
+ */
+
+ public static long getTransactorMaxCacheSize(FluoConfiguration conf) {
+ long size = conf.getLong(TRANSACTOR_MAX_CACHE_SIZE, TRANSACTOR_MAX_CACHE_SIZE_DEFAULT);
+ if (size <= 0) {
+ throw new IllegalArgumentException(
+ "Cache size must be positive for " + TRANSACTOR_MAX_CACHE_SIZE);
+ }
+ return size;
+ }
+
+ public static final String TRANSACTOR_CACHE_TIMEOUT =
+ FLUO_IMPL_PREFIX + ".transactor.cache.expireTime.ms";
+
+ public static final long TRANSACTOR_CACHE_TIMEOUT_DEFAULT = 24 * 60 * 1000;
+
+ public static long getTransactorCacheTimeout(FluoConfiguration conf, TimeUnit tu) {
+ long millis = conf.getLong(TRANSACTOR_CACHE_TIMEOUT, TRANSACTOR_CACHE_TIMEOUT_DEFAULT);
+ if (millis <= 0) {
+ throw new IllegalArgumentException("Timeout must positive for " + TRANSACTOR_CACHE_TIMEOUT);
+ }
+ return tu.convert(millis, TimeUnit.MILLISECONDS);
+ }
+
public static final String ASYNC_CW_THREADS = FLUO_IMPL_PREFIX + ".async.cw.threads";
public static final int ASYNC_CW_THREADS_DEFAULT = 8;
public static final String ASYNC_CW_LIMIT = FLUO_IMPL_PREFIX + ".async.cw.limit";
diff --git a/modules/core/src/main/java/org/apache/fluo/core/impl/TransactorCache.java b/modules/core/src/main/java/org/apache/fluo/core/impl/TransactorCache.java
index 892ba4a..2831329 100644
--- a/modules/core/src/main/java/org/apache/fluo/core/impl/TransactorCache.java
+++ b/modules/core/src/main/java/org/apache/fluo/core/impl/TransactorCache.java
@@ -27,6 +27,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCache;
import org.apache.curator.framework.recipes.cache.PathChildrenCache.StartMode;
import org.apache.fluo.accumulo.util.LongUtil;
import org.apache.fluo.accumulo.util.ZookeeperPath;
+import org.apache.fluo.api.config.FluoConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,11 +48,14 @@ public class TransactorCache implements AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(TransactorCache.class);
public TransactorCache(Environment env) {
-
- timeoutCache = CacheBuilder.newBuilder().maximumSize(1 << 15)
- .expireAfterAccess(FluoConfigurationImpl.TX_INFO_CACHE_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS)
- .concurrencyLevel(10).build();
+ final FluoConfiguration conf = env.getConfiguration();
+
+ timeoutCache =
+ CacheBuilder.newBuilder().maximumSize(FluoConfigurationImpl.getTransactorMaxCacheSize(conf))
+ .expireAfterAccess(
+ FluoConfigurationImpl.getTransactorCacheTimeout(conf, TimeUnit.MILLISECONDS),
+ TimeUnit.MILLISECONDS)
+ .concurrencyLevel(10).build();
this.env = env;
cache = new PathChildrenCache(env.getSharedResources().getCurator(),
diff --git a/modules/core/src/main/java/org/apache/fluo/core/impl/TxInfoCache.java b/modules/core/src/main/java/org/apache/fluo/core/impl/TxInfoCache.java
index 6d3a49e..3947c5c 100644
--- a/modules/core/src/main/java/org/apache/fluo/core/impl/TxInfoCache.java
+++ b/modules/core/src/main/java/org/apache/fluo/core/impl/TxInfoCache.java
@@ -41,7 +41,7 @@ public class TxInfoCache {
cache = CacheBuilder.newBuilder()
.expireAfterAccess(FluoConfigurationImpl.getTxIfoCacheTimeout(conf, TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS)
- .maximumWeight(FluoConfigurationImpl.getTxInfoCacheSize(conf))
+ .maximumWeight(FluoConfigurationImpl.getTxInfoCacheWeight(conf))
.weigher(new TxStatusWeigher()).concurrencyLevel(10).build();
this.env = env;
}
diff --git a/modules/core/src/main/java/org/apache/fluo/core/impl/VisibilityCache.java b/modules/core/src/main/java/org/apache/fluo/core/impl/VisibilityCache.java
index 4d75fbd..439b9a8 100644
--- a/modules/core/src/main/java/org/apache/fluo/core/impl/VisibilityCache.java
+++ b/modules/core/src/main/java/org/apache/fluo/core/impl/VisibilityCache.java
@@ -52,8 +52,8 @@ public class VisibilityCache {
.expireAfterAccess(
FluoConfigurationImpl.getVisibilityCacheTimeout(conf, TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS)
- .maximumWeight(FluoConfigurationImpl.getVisibilityCacheSize(conf)).weigher(new VisWeigher())
- .concurrencyLevel(10).build();
+ .maximumWeight(FluoConfigurationImpl.getVisibilityCacheWeight(conf))
+ .weigher(new VisWeigher()).concurrencyLevel(10).build();
}
public ColumnVisibility getCV(Column col) {
--
To stop receiving notification emails like this one, please contact
['"commits@fluo.apache.org" <co...@fluo.apache.org>'].