You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jm...@apache.org on 2015/09/11 19:08:02 UTC
cassandra git commit: Add configurable warning threshold for GC
duration
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 e9f36042c -> 488db6f50
Add configurable warning threshold for GC duration
Patch by jmckenzie and achowdhery for CASSANDRA-8907
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/488db6f5
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/488db6f5
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/488db6f5
Branch: refs/heads/cassandra-2.1
Commit: 488db6f50662e5e63c382c793d2c7ad69b8c74fc
Parents: e9f3604
Author: Joshua McKenzie <jm...@apache.org>
Authored: Fri Sep 11 12:54:30 2015 -0400
Committer: Joshua McKenzie <jm...@apache.org>
Committed: Fri Sep 11 12:54:30 2015 -0400
----------------------------------------------------------------------
CHANGES.txt | 1 +
conf/cassandra.yaml | 5 +++++
src/java/org/apache/cassandra/config/Config.java | 2 ++
.../apache/cassandra/config/DatabaseDescriptor.java | 11 +++++++++++
.../org/apache/cassandra/service/GCInspector.java | 16 +++++++++++-----
5 files changed, 30 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2b96949..4e0df42 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.1.10
+ * Added configurable warning threshold for GC duration (CASSANDRA-8907)
* (cqlsh) Make cqlsh PEP8 compliant (CASSANDRA-10066)
* (cqlsh) Fix error when starting cqlsh with --debug (CASSANDRA-10282)
* Scrub, Cleanup and Upgrade do not unmark compacting until all operations
http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/conf/cassandra.yaml
----------------------------------------------------------------------
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index 4e0111d..7e266d1 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -802,3 +802,8 @@ internode_compression: all
# reducing overhead from the TCP protocol itself, at the cost of increasing
# latency if you block for cross-datacenter responses.
inter_dc_tcp_nodelay: false
+
+# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level
+# Adjust the threshold based on your application throughput requirement
+# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level
+# gc_warn_threshold_in_ms: 1000
http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/src/java/org/apache/cassandra/config/Config.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java
index d960463..911dd73 100644
--- a/src/java/org/apache/cassandra/config/Config.java
+++ b/src/java/org/apache/cassandra/config/Config.java
@@ -228,6 +228,8 @@ public class Config
public volatile Long index_summary_capacity_in_mb;
public volatile int index_summary_resize_interval_in_minutes = 60;
+ public int gc_warn_threshold_in_ms = 0;
+
private static final CsvPreference STANDARD_SURROUNDING_SPACES_NEED_QUOTES = new CsvPreference.Builder(CsvPreference.STANDARD_PREFERENCE)
.surroundingSpacesNeedQuotes(true).build();
http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index bbecc6b..3a6a8fd 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -389,6 +389,11 @@ public class DatabaseDescriptor
}
paritionerName = partitioner.getClass().getCanonicalName();
+ if (conf.gc_warn_threshold_in_ms < 0)
+ {
+ throw new ConfigurationException("gc_warn_threshold_in_ms must be a positive integer");
+ }
+
if (conf.max_hint_window_in_ms == null)
{
throw new ConfigurationException("max_hint_window_in_ms cannot be set to null");
@@ -1707,4 +1712,10 @@ public class DatabaseDescriptor
{
return conf.otc_coalescing_window_us;
}
+
+ public static long getGCWarnThreshold()
+ {
+ return conf.gc_warn_threshold_in_ms;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/cassandra/blob/488db6f5/src/java/org/apache/cassandra/service/GCInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java
index df0527f..cf17a34 100644
--- a/src/java/org/apache/cassandra/service/GCInspector.java
+++ b/src/java/org/apache/cassandra/service/GCInspector.java
@@ -25,17 +25,20 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+
import javax.management.MBeanServer;
import javax.management.Notification;
import javax.management.NotificationListener;
import javax.management.ObjectName;
import javax.management.openmbean.CompositeData;
+import com.sun.management.GarbageCollectionNotificationInfo;
+import com.sun.management.GcInfo;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.sun.management.GarbageCollectionNotificationInfo;
-import com.sun.management.GcInfo;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
import org.apache.cassandra.utils.StatusLogger;
@@ -44,7 +47,8 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean
public static final String MBEAN_NAME = "org.apache.cassandra.service:type=GCInspector";
private static final Logger logger = LoggerFactory.getLogger(GCInspector.class);
final static long MIN_LOG_DURATION = 200;
- final static long MIN_LOG_DURATION_TPSTATS = 1000;
+ final static long GC_WARN_THRESHOLD_IN_MS = DatabaseDescriptor.getGCWarnThreshold();
+ final static long STAT_THRESHOLD = Math.min(GC_WARN_THRESHOLD_IN_MS != 0 ? GC_WARN_THRESHOLD_IN_MS : MIN_LOG_DURATION, MIN_LOG_DURATION);
static final class State
{
@@ -248,12 +252,14 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean
}
String st = sb.toString();
- if (duration > MIN_LOG_DURATION)
+ if (GC_WARN_THRESHOLD_IN_MS != 0 && duration > GC_WARN_THRESHOLD_IN_MS)
+ logger.warn(st);
+ else if (duration > MIN_LOG_DURATION)
logger.info(st);
else if (logger.isDebugEnabled())
logger.debug(st);
- if (duration > MIN_LOG_DURATION_TPSTATS)
+ if (duration > STAT_THRESHOLD)
StatusLogger.log();
// if we just finished an old gen collection and we're still using a lot of memory, try to reduce the pressure