You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2014/01/30 17:16:38 UTC
[1/3] git commit: revert CASSANDRA-6107,
and cut max cached statements to 50k
Updated Branches:
refs/heads/cassandra-1.2 6c4333e3d -> 852e27f2e
refs/heads/cassandra-2.0 39bdb2be9 -> cd91cf974
revert CASSANDRA-6107, and cut max cached statements to 50k
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/852e27f2
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/852e27f2
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/852e27f2
Branch: refs/heads/cassandra-1.2
Commit: 852e27f2e8f8d6d1e785baf0d834ebfd1d60ca27
Parents: 6c4333e
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Jan 30 10:16:08 2014 -0600
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Jan 30 10:16:15 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/cql3/QueryProcessor.java | 81 +++-----------------
2 files changed, 10 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/852e27f2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d88bb26..110bf50 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.2.14
+ * Reverted code to limit CQL prepared statement cache by size (CASSANDRA-6592)
* add cassandra.default_messaging_version property to allow easier
upgrading from 1.1 (CASSANDRA-6619)
* Allow executing CREATE statements multiple times (CASSANDRA-6471)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/852e27f2/src/java/org/apache/cassandra/cql3/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
index dc94a18..67ecefc 100644
--- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java
+++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
@@ -20,12 +20,8 @@ package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
import java.util.*;
-import com.google.common.primitives.Ints;
-
import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.googlecode.concurrentlinkedhashmap.EntryWeigher;
import org.antlr.runtime.*;
-import org.github.jamm.MemoryMeter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,56 +41,15 @@ public class QueryProcessor
public static final SemanticVersion CQL_VERSION = new SemanticVersion("3.0.5");
private static final Logger logger = LoggerFactory.getLogger(QueryProcessor.class);
- private static final MemoryMeter meter = new MemoryMeter();
- private static final long MAX_CACHE_PREPARED_MEMORY = Runtime.getRuntime().maxMemory() / 256;
- private static final int MAX_CACHE_PREPARED_COUNT = 10000;
-
- private static EntryWeigher<MD5Digest, CQLStatement> cqlMemoryUsageWeigher = new EntryWeigher<MD5Digest, CQLStatement>()
- {
- @Override
- public int weightOf(MD5Digest key, CQLStatement value)
- {
- return Ints.checkedCast(measure(key) + measure(value));
- }
- };
- private static EntryWeigher<Integer, CQLStatement> thriftMemoryUsageWeigher = new EntryWeigher<Integer, CQLStatement>()
- {
- @Override
- public int weightOf(Integer key, CQLStatement value)
- {
- return Ints.checkedCast(measure(key) + measure(value));
- }
- };
+ public static final int MAX_CACHE_PREPARED = 50000; // Enough to keep buggy clients from OOM'ing us
+ private static final Map<MD5Digest, CQLStatement> preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, CQLStatement>()
+ .maximumWeightedCapacity(MAX_CACHE_PREPARED)
+ .build();
- private static final ConcurrentLinkedHashMap<MD5Digest, CQLStatement> preparedStatements;
- private static final ConcurrentLinkedHashMap<Integer, CQLStatement> thriftPreparedStatements;
-
- static
- {
- if (MemoryMeter.isInitialized())
- {
- preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY)
- .weigher(cqlMemoryUsageWeigher)
- .build();
- thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY)
- .weigher(thriftMemoryUsageWeigher)
- .build();
- }
- else
- {
- logger.error("Unable to initialize MemoryMeter (jamm not specified as javaagent). This means "
- + "Cassandra will be unable to measure object sizes accurately and may consequently OOM.");
- preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_COUNT)
- .build();
- thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_COUNT)
- .build();
- }
- }
+ private static final Map<Integer, CQLStatement> thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, CQLStatement>()
+ .maximumWeightedCapacity(MAX_CACHE_PREPARED)
+ .build();
public static CQLStatement getPrepared(MD5Digest id)
{
@@ -228,18 +183,10 @@ public class QueryProcessor
}
private static ResultMessage.Prepared storePreparedStatement(String queryString, String keyspace, ParsedStatement.Prepared prepared, boolean forThrift)
- throws InvalidRequestException
{
// Concatenate the current keyspace so we don't mix prepared statements between keyspace (#5352).
// (if the keyspace is null, queryString has to have a fully-qualified keyspace so it's fine.
String toHash = keyspace == null ? queryString : keyspace + queryString;
- long statementSize = measure(prepared.statement);
- // don't execute the statement if it's bigger than the allowed threshold
- if (statementSize > MAX_CACHE_PREPARED_MEMORY)
- throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d bytes.",
- statementSize,
- MAX_CACHE_PREPARED_MEMORY));
-
if (forThrift)
{
int statementId = toHash.hashCode();
@@ -254,8 +201,8 @@ public class QueryProcessor
MD5Digest statementId = MD5Digest.compute(toHash);
preparedStatements.put(statementId, prepared.statement);
logger.trace(String.format("Stored prepared statement %s with %d bind markers",
- statementId,
- prepared.statement.getBoundTerms()));
+ statementId,
+ prepared.statement.getBoundTerms()));
return new ResultMessage.Prepared(statementId, prepared.boundNames);
}
}
@@ -327,14 +274,4 @@ public class QueryProcessor
throw new SyntaxException("Invalid or malformed CQL query string: " + e.getMessage());
}
}
-
- private static long measure(Object key)
- {
- if (!MemoryMeter.isInitialized())
- return 1;
-
- return key instanceof MeasurableForPreparedCache
- ? ((MeasurableForPreparedCache)key).measureForPreparedCache(meter)
- : meter.measureDeep(key);
- }
}
[3/3] git commit: Merge branch 'cassandra-1.2' into cassandra-2.0
Posted by jb...@apache.org.
Merge branch 'cassandra-1.2' into cassandra-2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cd91cf97
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cd91cf97
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cd91cf97
Branch: refs/heads/cassandra-2.0
Commit: cd91cf9746adf93788aa2cca76f8430e33216300
Parents: 39bdb2b 852e27f
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Jan 30 10:16:27 2014 -0600
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Jan 30 10:16:27 2014 -0600
----------------------------------------------------------------------
----------------------------------------------------------------------
[2/3] git commit: revert CASSANDRA-6107,
and cut max cached statements to 50k
Posted by jb...@apache.org.
revert CASSANDRA-6107, and cut max cached statements to 50k
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/852e27f2
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/852e27f2
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/852e27f2
Branch: refs/heads/cassandra-2.0
Commit: 852e27f2e8f8d6d1e785baf0d834ebfd1d60ca27
Parents: 6c4333e
Author: Jonathan Ellis <jb...@apache.org>
Authored: Thu Jan 30 10:16:08 2014 -0600
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Thu Jan 30 10:16:15 2014 -0600
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/cql3/QueryProcessor.java | 81 +++-----------------
2 files changed, 10 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/852e27f2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d88bb26..110bf50 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
1.2.14
+ * Reverted code to limit CQL prepared statement cache by size (CASSANDRA-6592)
* add cassandra.default_messaging_version property to allow easier
upgrading from 1.1 (CASSANDRA-6619)
* Allow executing CREATE statements multiple times (CASSANDRA-6471)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/852e27f2/src/java/org/apache/cassandra/cql3/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
index dc94a18..67ecefc 100644
--- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java
+++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java
@@ -20,12 +20,8 @@ package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
import java.util.*;
-import com.google.common.primitives.Ints;
-
import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
-import com.googlecode.concurrentlinkedhashmap.EntryWeigher;
import org.antlr.runtime.*;
-import org.github.jamm.MemoryMeter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,56 +41,15 @@ public class QueryProcessor
public static final SemanticVersion CQL_VERSION = new SemanticVersion("3.0.5");
private static final Logger logger = LoggerFactory.getLogger(QueryProcessor.class);
- private static final MemoryMeter meter = new MemoryMeter();
- private static final long MAX_CACHE_PREPARED_MEMORY = Runtime.getRuntime().maxMemory() / 256;
- private static final int MAX_CACHE_PREPARED_COUNT = 10000;
-
- private static EntryWeigher<MD5Digest, CQLStatement> cqlMemoryUsageWeigher = new EntryWeigher<MD5Digest, CQLStatement>()
- {
- @Override
- public int weightOf(MD5Digest key, CQLStatement value)
- {
- return Ints.checkedCast(measure(key) + measure(value));
- }
- };
- private static EntryWeigher<Integer, CQLStatement> thriftMemoryUsageWeigher = new EntryWeigher<Integer, CQLStatement>()
- {
- @Override
- public int weightOf(Integer key, CQLStatement value)
- {
- return Ints.checkedCast(measure(key) + measure(value));
- }
- };
+ public static final int MAX_CACHE_PREPARED = 50000; // Enough to keep buggy clients from OOM'ing us
+ private static final Map<MD5Digest, CQLStatement> preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, CQLStatement>()
+ .maximumWeightedCapacity(MAX_CACHE_PREPARED)
+ .build();
- private static final ConcurrentLinkedHashMap<MD5Digest, CQLStatement> preparedStatements;
- private static final ConcurrentLinkedHashMap<Integer, CQLStatement> thriftPreparedStatements;
-
- static
- {
- if (MemoryMeter.isInitialized())
- {
- preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY)
- .weigher(cqlMemoryUsageWeigher)
- .build();
- thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_MEMORY)
- .weigher(thriftMemoryUsageWeigher)
- .build();
- }
- else
- {
- logger.error("Unable to initialize MemoryMeter (jamm not specified as javaagent). This means "
- + "Cassandra will be unable to measure object sizes accurately and may consequently OOM.");
- preparedStatements = new ConcurrentLinkedHashMap.Builder<MD5Digest, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_COUNT)
- .build();
- thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, CQLStatement>()
- .maximumWeightedCapacity(MAX_CACHE_PREPARED_COUNT)
- .build();
- }
- }
+ private static final Map<Integer, CQLStatement> thriftPreparedStatements = new ConcurrentLinkedHashMap.Builder<Integer, CQLStatement>()
+ .maximumWeightedCapacity(MAX_CACHE_PREPARED)
+ .build();
public static CQLStatement getPrepared(MD5Digest id)
{
@@ -228,18 +183,10 @@ public class QueryProcessor
}
private static ResultMessage.Prepared storePreparedStatement(String queryString, String keyspace, ParsedStatement.Prepared prepared, boolean forThrift)
- throws InvalidRequestException
{
// Concatenate the current keyspace so we don't mix prepared statements between keyspace (#5352).
// (if the keyspace is null, queryString has to have a fully-qualified keyspace so it's fine.
String toHash = keyspace == null ? queryString : keyspace + queryString;
- long statementSize = measure(prepared.statement);
- // don't execute the statement if it's bigger than the allowed threshold
- if (statementSize > MAX_CACHE_PREPARED_MEMORY)
- throw new InvalidRequestException(String.format("Prepared statement of size %d bytes is larger than allowed maximum of %d bytes.",
- statementSize,
- MAX_CACHE_PREPARED_MEMORY));
-
if (forThrift)
{
int statementId = toHash.hashCode();
@@ -254,8 +201,8 @@ public class QueryProcessor
MD5Digest statementId = MD5Digest.compute(toHash);
preparedStatements.put(statementId, prepared.statement);
logger.trace(String.format("Stored prepared statement %s with %d bind markers",
- statementId,
- prepared.statement.getBoundTerms()));
+ statementId,
+ prepared.statement.getBoundTerms()));
return new ResultMessage.Prepared(statementId, prepared.boundNames);
}
}
@@ -327,14 +274,4 @@ public class QueryProcessor
throw new SyntaxException("Invalid or malformed CQL query string: " + e.getMessage());
}
}
-
- private static long measure(Object key)
- {
- if (!MemoryMeter.isInitialized())
- return 1;
-
- return key instanceof MeasurableForPreparedCache
- ? ((MeasurableForPreparedCache)key).measureForPreparedCache(meter)
- : meter.measureDeep(key);
- }
}