You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by sn...@apache.org on 2016/05/02 17:05:44 UTC
cassandra git commit: Make prepared statement cache size configurable
Repository: cassandra
Updated Branches:
refs/heads/trunk 3513fbcfb -> 9581209b3
Make prepared statement cache size configurable
patch by Robert Stupp; reviewed by Benjamin Lerer for CASSANDRA-11555
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9581209b
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9581209b
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9581209b
Branch: refs/heads/trunk
Commit: 9581209b35922a0758c6bd158d4336a17cfe86aa
Parents: 3513fbc
Author: Robert Stupp <sn...@snazy.de>
Authored: Mon May 2 17:03:56 2016 +0200
Committer: Robert Stupp <sn...@snazy.de>
Committed: Mon May 2 17:03:56 2016 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 28 ++++++++
.../org/apache/cassandra/config/Config.java | 11 +++
.../cassandra/config/DatabaseDescriptor.java | 45 ++++++++++++
.../apache/cassandra/cql3/QueryProcessor.java | 72 +++++++++++---------
5 files changed, 125 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c49249c..c802031 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
3.6
+ * Make prepared statement cache size configurable (CASSANDRA-11555)
* Integrated JMX authentication and authorization (CASSANDRA-10091)
* Add units to stress ouput (CASSANDRA-11352)
* Fix PER PARTITION LIMIT for single and multi partitions queries (CASSANDRA-11603)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 48bad2c..9eb55e1 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -215,6 +215,34 @@ disk_failure_policy: stop
# ignore: ignore fatal errors and let the batches fail
commit_failure_policy: stop
+# Maximum size of the native protocol prepared statement cache
+#
+# Valid values are either "auto" (omitting the value) or a value greater 0.
+#
+# Note that specifying a too large value will result in long running GCs and possbily
+# out-of-memory errors. Keep the value at a small fraction of the heap.
+#
+# If you constantly see "prepared statements discarded in the last minute because
+# cache limit reached" messages, the first step is to investigate the root cause
+# of these messages and check whether prepared statements are used correctly -
+# i.e. use bind markers for variable parts.
+#
+# Do only change the default value, if you really have more prepared statements than
+# fit in the cache. In most cases it is not neccessary to change this value.
+# Constantly re-preparing statements is a performance penalty.
+#
+# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater
+prepared_statements_cache_size_mb:
+
+# Maximum size of the Thrift prepared statement cache
+#
+# If you do not use Thrift at all, it is safe to leave this value at "auto".
+#
+# See description of 'prepared_statements_cache_size_mb' above for more information.
+#
+# Default value ("auto") is 1/256th of the heap or 10MB, whichever is greater
+thrift_prepared_statements_cache_size_mb:
+
# Maximum size of the key cache in memory.
#
# Each key cache hit saves 1 seek and each row cache hit saves 2 seeks at the
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index 809966d..02635bf 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -288,6 +288,17 @@ public class Config
public int windows_timer_interval = 0;
+ /**
+ * Size of the CQL prepared statements cache in MB.
+ * Defaults to 1/256th of the heap size or 10MB, whichever is greater.
+ */
+ public Long prepared_statements_cache_size_mb = null;
+ /**
+ * Size of the Thrift prepared statements cache in MB.
+ * Defaults to 1/256th of the heap size or 10MB, whichever is greater.
+ */
+ public Long thrift_prepared_statements_cache_size_mb = null;
+
public boolean enable_user_defined_functions = false;
public boolean enable_scripted_user_defined_functions = false;
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index b98d103..d8acdb8 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -96,6 +96,9 @@ public class DatabaseDescriptor
private static RequestSchedulerId requestSchedulerId;
private static RequestSchedulerOptions requestSchedulerOptions;
+ private static long preparedStatementsCacheSizeInMB;
+ private static long thriftPreparedStatementsCacheSizeInMB;
+
private static long keyCacheSizeInMB;
private static long counterCacheSizeInMB;
private static long indexSummaryCapacityInMB;
@@ -641,6 +644,38 @@ public class DatabaseDescriptor
try
{
+ // if prepared_statements_cache_size_mb option was set to "auto" then size of the cache should be "max(1/256 of Heap (in MB), 10MB)"
+ preparedStatementsCacheSizeInMB = (conf.prepared_statements_cache_size_mb == null)
+ ? Math.max(10, (int) (Runtime.getRuntime().maxMemory() / 1024 / 1024 / 256))
+ : conf.prepared_statements_cache_size_mb;
+
+ if (preparedStatementsCacheSizeInMB <= 0)
+ throw new NumberFormatException(); // to escape duplicating error message
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ConfigurationException("prepared_statements_cache_size_mb option was set incorrectly to '"
+ + conf.prepared_statements_cache_size_mb + "', supported values are <integer> >= 0.", false);
+ }
+
+ try
+ {
+ // if thrift_prepared_statements_cache_size_mb option was set to "auto" then size of the cache should be "max(1/256 of Heap (in MB), 10MB)"
+ thriftPreparedStatementsCacheSizeInMB = (conf.thrift_prepared_statements_cache_size_mb == null)
+ ? Math.max(10, (int) (Runtime.getRuntime().maxMemory() / 1024 / 1024 / 256))
+ : conf.thrift_prepared_statements_cache_size_mb;
+
+ if (thriftPreparedStatementsCacheSizeInMB <= 0)
+ throw new NumberFormatException(); // to escape duplicating error message
+ }
+ catch (NumberFormatException e)
+ {
+ throw new ConfigurationException("thrift_prepared_statements_cache_size_mb option was set incorrectly to '"
+ + conf.thrift_prepared_statements_cache_size_mb + "', supported values are <integer> >= 0.", false);
+ }
+
+ try
+ {
// if key_cache_size_in_mb option was set to "auto" then size of the cache should be "min(5% of Heap (in MB), 100MB)
keyCacheSizeInMB = (conf.key_cache_size_in_mb == null)
? Math.min(Math.max(1, (int) (Runtime.getRuntime().totalMemory() * 0.05 / 1024 / 1024)), 100)
@@ -1992,6 +2027,16 @@ public class DatabaseDescriptor
return conf.windows_timer_interval;
}
+ public static long getPreparedStatementsCacheSizeMB()
+ {
+ return preparedStatementsCacheSizeInMB;
+ }
+
+ public static long getThriftPreparedStatementsCacheSizeMB()
+ {
+ return thriftPreparedStatementsCacheSizeInMB;
+ }
+
public static boolean enableUserDefinedFunctions()
{
return conf.enable_user_defined_functions;
http://git-wip-us.apache.org/repos/asf/cassandra/blob/9581209b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
index 5f4b0f6..39a9c03 100644
--- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java
+++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
@@ -34,9 +34,9 @@ import org.slf4j.LoggerFactory;
import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
import com.googlecode.concurrentlinkedhashmap.EntryWeigher;
-import com.googlecode.concurrentlinkedhashmap.EvictionListener;
import org.antlr.runtime.*;
import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.Schema;
import org.apache.cassandra.cql3.functions.Function;
import org.apache.cassandra.cql3.functions.FunctionName;
@@ -65,7 +65,6 @@ public class QueryProcessor implements QueryHandler
private static final Logger logger = LoggerFactory.getLogger(QueryProcessor.class);
private static final MemoryMeter meter = new MemoryMeter().withGuessing(MemoryMeter.Guess.FALLBACK_BEST).ignoreKnownSingletons();
- private static final long MAX_CACHE_PREPARED_MEMORY = Runtime.getRuntime().maxMemory() / 256;
private static final EntryWeigher<MD5Digest, ParsedStatement.Prepared> cqlMemoryUsageWeigher = new EntryWeigher<MD5Digest, ParsedStatement.Prepared>()
{
@@ -97,45 +96,48 @@ public class QueryProcessor implements QueryHandler
public static final CQLMetrics metrics = new CQLMetrics();
private static final AtomicInteger lastMinuteEvictionsCount = new AtomicInteger(0);
+ private static final AtomicInteger thriftLastMinuteEvictionsCount = new AtomicInteger(0);
static
{
preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, ParsedStatement.Prepared>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY)
+ .maximumWeightedCapacity(capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB()))
.weigher(cqlMemoryUsageWeigher)
- .listener(new EvictionListener<MD5Digest, ParsedStatement.Prepared>()
- {
- public void onEviction(MD5Digest md5Digest, ParsedStatement.Prepared prepared)
- {
- metrics.preparedStatementsEvicted.inc();
- lastMinuteEvictionsCount.incrementAndGet();
- }
+ .listener((md5Digest, prepared) -> {
+ metrics.preparedStatementsEvicted.inc();
+ lastMinuteEvictionsCount.incrementAndGet();
}).build();
thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, ParsedStatement.Prepared>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY)
+ .maximumWeightedCapacity(capacityToBytes(DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB()))
.weigher(thriftMemoryUsageWeigher)
- .listener(new EvictionListener<Integer, ParsedStatement.Prepared>()
- {
- public void onEviction(Integer integer, ParsedStatement.Prepared prepared)
- {
- metrics.preparedStatementsEvicted.inc();
- lastMinuteEvictionsCount.incrementAndGet();
- }
+ .listener((integer, prepared) -> {
+ metrics.preparedStatementsEvicted.inc();
+ thriftLastMinuteEvictionsCount.incrementAndGet();
})
.build();
- ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(new Runnable()
- {
- public void run()
- {
- long count = lastMinuteEvictionsCount.getAndSet(0);
- if (count > 0)
- logger.info("{} prepared statements discarded in the last minute because cache limit reached ({})",
- count,
- FBUtilities.prettyPrintMemory(MAX_CACHE_PREPARED_MEMORY));
- }
+ ScheduledExecutors.scheduledTasks.scheduleAtFixedRate(() -> {
+ long count = lastMinuteEvictionsCount.getAndSet(0);
+ if (count > 0)
+ logger.warn("{} prepared statements discarded in the last minute because cache limit reached ({} MB)",
+ count,
+ DatabaseDescriptor.getPreparedStatementsCacheSizeMB());
+ count = thriftLastMinuteEvictionsCount.getAndSet(0);
+ if (count > 0)
+ logger.warn("{} prepared Thrift statements discarded in the last minute because cache limit reached ({} MB)",
+ count,
+ DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB());
}, 1, 1, TimeUnit.MINUTES);
+
+ logger.info("Initialized prepared statement caches with {} MB (native) and {} MB (Thrift)",
+ DatabaseDescriptor.getPreparedStatementsCacheSizeMB(),
+ DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB());
+ }
+
+ private static long capacityToBytes(long cacheSizeMB)
+ {
+ return cacheSizeMB * 1024 * 1024;
}
public static int preparedStatementsCount()
@@ -428,18 +430,24 @@ public class QueryProcessor implements QueryHandler
// (if the keyspace is null, queryString has to have a fully-qualified keyspace so it's fine.
long statementSize = measure(prepared.statement);
// don't execute the statement if it's bigger than the allowed threshold
- if (statementSize > MAX_CACHE_PREPARED_MEMORY)
- throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d bytes.",
- statementSize,
- MAX_CACHE_PREPARED_MEMORY));
if (forThrift)
{
+ if (statementSize > capacityToBytes(DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB()))
+ throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...",
+ statementSize,
+ DatabaseDescriptor.getThriftPreparedStatementsCacheSizeMB(),
+ queryString.substring(0, 200)));
Integer statementId = computeThriftId(queryString, keyspace);
thriftPreparedStatements.put(statementId, prepared);
return ResultMessage.Prepared.forThrift(statementId, prepared.boundNames);
}
else
{
+ if (statementSize > capacityToBytes(DatabaseDescriptor.getPreparedStatementsCacheSizeMB()))
+ throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d MB: %s...",
+ statementSize,
+ DatabaseDescriptor.getPreparedStatementsCacheSizeMB(),
+ queryString.substring(0, 200)));
MD5Digest statementId = computeId(queryString, keyspace);
preparedStatements.put(statementId, prepared);
return new ResultMessage.Prepared(statementId, prepared);