You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by jb...@apache.org on 2014/07/23 21:56:06 UTC
[1/7] git commit: Catch errors when the JVM pulls the rug out from
GCInspector patch by Josh McKenzie; reviewed by jbellis for CASSANDRA-5345
Repository: cassandra
Updated Branches:
refs/heads/cassandra-2.1 f9fc846e9 -> f7e3b9a6b
refs/heads/trunk 6076f8ff3 -> 432cfeff1
Catch errors when the JVM pulls the rug out from GCInspector
patch by Josh McKenzie; reviewed by jbellis for CASSANDRA-5345
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/51a1f232
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/51a1f232
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/51a1f232
Branch: refs/heads/cassandra-2.1
Commit: 51a1f232b42b4385df7f2aca42a97b4c5b02c077
Parents: 062addb
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Jul 18 18:13:33 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Jul 18 18:13:33 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/service/GCInspector.java | 130 +++++++++++--------
2 files changed, 79 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/51a1f232/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d339309..d4b11d1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.10
+ * Catch errors when the JVM pulls the rug out from GCInspector (CASSANDRA-5345)
* (Windows) force range-based repair to non-sequential mode (CASSANDRA-7541)
* Fix range merging when DES scores are zero (CASSANDRA-7535)
* Warn when SSL certificates have expired (CASSANDRA-7528)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/51a1f232/src/java/org/apache/cassandra/service/GCInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java
index 9961bf9..f03ec01 100644
--- a/src/java/org/apache/cassandra/service/GCInspector.java
+++ b/src/java/org/apache/cassandra/service/GCInspector.java
@@ -21,6 +21,7 @@ import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
+import java.lang.reflect.UndeclaredThrowableException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -49,8 +50,28 @@ public class GCInspector
final List<GarbageCollectorMXBean> beans = new ArrayList<GarbageCollectorMXBean>();
final MemoryMXBean membean = ManagementFactory.getMemoryMXBean();
- public GCInspector()
+ public void start()
{
+ buildMXBeanList();
+
+ // don't bother starting a thread that will do nothing.
+ if (beans.isEmpty())
+ return;
+
+ Runnable t = new Runnable()
+ {
+ public void run()
+ {
+ logGCResults();
+ }
+ };
+ StorageService.scheduledTasks.scheduleWithFixedDelay(t, INTERVAL_IN_MS, INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
+ }
+
+ private void buildMXBeanList()
+ {
+ beans.clear();
+
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
try
{
@@ -67,62 +88,67 @@ public class GCInspector
}
}
- public void start()
+ private void logGCResults()
{
- // don't bother starting a thread that will do nothing.
- if (beans.size() == 0)
- return;
- Runnable t = new Runnable()
+ boolean gcChanged = false;
+ try
{
- public void run()
+ for (GarbageCollectorMXBean gc : beans)
{
- logGCResults();
+ if (!gc.isValid())
+ {
+ gcChanged = true;
+ continue;
+ }
+
+ Long previousTotal = gctimes.get(gc.getName());
+ Long total = gc.getCollectionTime();
+ if (previousTotal == null)
+ previousTotal = 0L;
+ if (previousTotal.equals(total))
+ continue;
+ gctimes.put(gc.getName(), total);
+ Long duration = total - previousTotal; // may be zero for a really fast collection
+
+ Long previousCount = gccounts.get(gc.getName());
+ Long count = gc.getCollectionCount();
+
+ if (previousCount == null)
+ previousCount = 0L;
+ if (count.equals(previousCount))
+ continue;
+
+ gccounts.put(gc.getName(), count);
+
+ MemoryUsage mu = membean.getHeapMemoryUsage();
+ long memoryUsed = mu.getUsed();
+ long memoryMax = mu.getMax();
+
+ String st = String.format("GC for %s: %s ms for %s collections, %s used; max is %s",
+ gc.getName(), duration, count - previousCount, memoryUsed, memoryMax);
+ long durationPerCollection = duration / (count - previousCount);
+ if (durationPerCollection > MIN_DURATION)
+ logger.info(st);
+ else if (logger.isDebugEnabled())
+ logger.debug(st);
+
+ if (durationPerCollection > MIN_DURATION_TPSTATS)
+ StatusLogger.log();
+
+ // if we just finished a full collection and we're still using a lot of memory, try to reduce the pressure
+ if (gc.getName().equals("ConcurrentMarkSweep"))
+ SSTableDeletingTask.rescheduleFailedTasks();
}
- };
- StorageService.scheduledTasks.scheduleWithFixedDelay(t, INTERVAL_IN_MS, INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
- }
-
- private void logGCResults()
- {
- for (GarbageCollectorMXBean gc : beans)
+ }
+ catch (UndeclaredThrowableException e)
{
- Long previousTotal = gctimes.get(gc.getName());
- Long total = gc.getCollectionTime();
- if (previousTotal == null)
- previousTotal = 0L;
- if (previousTotal.equals(total))
- continue;
- gctimes.put(gc.getName(), total);
- Long duration = total - previousTotal; // may be zero for a really fast collection
-
- Long previousCount = gccounts.get(gc.getName());
- Long count = gc.getCollectionCount();
-
- if (previousCount == null)
- previousCount = 0L;
- if (count.equals(previousCount))
- continue;
-
- gccounts.put(gc.getName(), count);
-
- MemoryUsage mu = membean.getHeapMemoryUsage();
- long memoryUsed = mu.getUsed();
- long memoryMax = mu.getMax();
-
- String st = String.format("GC for %s: %s ms for %s collections, %s used; max is %s",
- gc.getName(), duration, count - previousCount, memoryUsed, memoryMax);
- long durationPerCollection = duration / (count - previousCount);
- if (durationPerCollection > MIN_DURATION)
- logger.info(st);
- else if (logger.isDebugEnabled())
- logger.debug(st);
-
- if (durationPerCollection > MIN_DURATION_TPSTATS)
- StatusLogger.log();
-
- // if we just finished a full collection and we're still using a lot of memory, try to reduce the pressure
- if (gc.getName().equals("ConcurrentMarkSweep"))
- SSTableDeletingTask.rescheduleFailedTasks();
+ // valid-ness may have changed out from under us, even though we check for it explicitly.
+ // if so, gc.getName() will throw UTE when reflection runs into InstanceNotFoundException.
+ // See CASSANDRA-5345
+ gcChanged = true;
}
+
+ if (gcChanged)
+ buildMXBeanList();
}
}
[5/7] git commit: Merge branch 'cassandra-2.1' into trunk
Posted by jb...@apache.org.
Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/eb63768a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/eb63768a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/eb63768a
Branch: refs/heads/trunk
Commit: eb63768a50b7d6ad32a74250851e3653bafad0fd
Parents: db70f5a b181261
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Jul 18 18:14:56 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Jul 18 18:14:56 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/service/GCInspector.java | 130 +++++++++++--------
2 files changed, 80 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/eb63768a/CHANGES.txt
----------------------------------------------------------------------
[4/7] git commit: merge from 2.0
Posted by jb...@apache.org.
merge from 2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b181261c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b181261c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b181261c
Branch: refs/heads/cassandra-2.1
Commit: b181261cfe2c975981c7a29a6b8c7ab4f5a7629f
Parents: 5dfee58 51a1f23
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Jul 18 18:14:47 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Jul 18 18:14:47 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/service/GCInspector.java | 130 +++++++++++--------
2 files changed, 80 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b181261c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index eb2a1a0,d4b11d1..a30de32
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,26 -1,5 +1,28 @@@
-2.0.10
+2.1.1
+ * Add listen_interface and rpc_interface options (CASSANDRA-7417)
+ * Fail to start if commit log replay detects a problem (CASSANDRA-7125)
+ * Improve schema merge performance (CASSANDRA-7444)
+ * Adjust MT depth based on # of partition validating (CASSANDRA-5263)
+ * Optimise NativeCell comparisons (CASSANDRA-6755)
+ * Configurable client timeout for cqlsh (CASSANDRA-7516)
+ * Include snippet of CQL query near syntax error in messages (CASSANDRA-7111)
++Merged from 2.0:
+ * Catch errors when the JVM pulls the rug out from GCInspector (CASSANDRA-5345)
+
+
+2.1.0-final
+ * Updated memtable_cleanup_threshold and memtable_flush_writers defaults
+ (CASSANDRA-7551)
+ * (Windows) fix startup when WMI memory query fails (CASSANDRA-7505)
+ * Anti-compaction proceeds if any part of the repair failed (CASANDRA-7521)
+ * Add missing table name to DROP INDEX responses and notifications (CASSANDRA-7539)
+ * Bump CQL version to 3.2.0 and update CQL documentation (CASSANDRA-7527)
+ * Fix configuration error message when running nodetool ring (CASSANDRA-7508)
+ * Support conditional updates, tuple type, and the v3 protocol in cqlsh (CASSANDRA-7509)
+ * Handle queries on multiple secondary index types (CASSANDRA-7525)
+ * Fix cqlsh authentication with v3 native protocol (CASSANDRA-7564)
+ * Fix NPE when unknown prepared statement ID is used (CASSANDRA-7454)
+Merged from 2.0:
* (Windows) force range-based repair to non-sequential mode (CASSANDRA-7541)
* Fix range merging when DES scores are zero (CASSANDRA-7535)
* Warn when SSL certificates have expired (CASSANDRA-7528)
[3/7] git commit: merge from 2.0
Posted by jb...@apache.org.
merge from 2.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/b181261c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/b181261c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/b181261c
Branch: refs/heads/trunk
Commit: b181261cfe2c975981c7a29a6b8c7ab4f5a7629f
Parents: 5dfee58 51a1f23
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Jul 18 18:14:47 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Jul 18 18:14:47 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../apache/cassandra/service/GCInspector.java | 130 +++++++++++--------
2 files changed, 80 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/b181261c/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index eb2a1a0,d4b11d1..a30de32
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,26 -1,5 +1,28 @@@
-2.0.10
+2.1.1
+ * Add listen_interface and rpc_interface options (CASSANDRA-7417)
+ * Fail to start if commit log replay detects a problem (CASSANDRA-7125)
+ * Improve schema merge performance (CASSANDRA-7444)
+ * Adjust MT depth based on # of partition validating (CASSANDRA-5263)
+ * Optimise NativeCell comparisons (CASSANDRA-6755)
+ * Configurable client timeout for cqlsh (CASSANDRA-7516)
+ * Include snippet of CQL query near syntax error in messages (CASSANDRA-7111)
++Merged from 2.0:
+ * Catch errors when the JVM pulls the rug out from GCInspector (CASSANDRA-5345)
+
+
+2.1.0-final
+ * Updated memtable_cleanup_threshold and memtable_flush_writers defaults
+ (CASSANDRA-7551)
+ * (Windows) fix startup when WMI memory query fails (CASSANDRA-7505)
+ * Anti-compaction proceeds if any part of the repair failed (CASANDRA-7521)
+ * Add missing table name to DROP INDEX responses and notifications (CASSANDRA-7539)
+ * Bump CQL version to 3.2.0 and update CQL documentation (CASSANDRA-7527)
+ * Fix configuration error message when running nodetool ring (CASSANDRA-7508)
+ * Support conditional updates, tuple type, and the v3 protocol in cqlsh (CASSANDRA-7509)
+ * Handle queries on multiple secondary index types (CASSANDRA-7525)
+ * Fix cqlsh authentication with v3 native protocol (CASSANDRA-7564)
+ * Fix NPE when unknown prepared statement ID is used (CASSANDRA-7454)
+Merged from 2.0:
* (Windows) force range-based repair to non-sequential mode (CASSANDRA-7541)
* Fix range merging when DES scores are zero (CASSANDRA-7535)
* Warn when SSL certificates have expired (CASSANDRA-7528)
[6/7] git commit: merge
Posted by jb...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/432cfeff
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/432cfeff
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/432cfeff
Branch: refs/heads/trunk
Commit: 432cfeff19c79cd278d30be984ea803c484cc867
Parents: eb63768 6076f8f
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Jul 23 12:55:49 2014 -0700
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Jul 23 12:55:49 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 8 +-
bin/cqlsh | 3 +-
build.xml | 2 +-
conf/cassandra-env.sh | 4 +-
debian/changelog | 6 +
doc/cql3/CQL.textile | 2 +-
examples/hadoop_cql3_word_count/README.txt | 6 +-
.../conf/log4j.properties | 2 +-
.../src/WordCountSetup.java | 5 +-
examples/hadoop_word_count/README.txt | 6 +-
.../hadoop_word_count/conf/log4j.properties | 2 +-
examples/hadoop_word_count/src/WordCount.java | 5 +-
.../src/WordCountCounters.java | 2 +
.../hadoop_word_count/src/WordCountSetup.java | 25 +-
pylib/cqlshlib/cql3handling.py | 5 +-
.../org/apache/cassandra/db/DefsTables.java | 2 +-
.../cassandra/hadoop/pig/CqlNativeStorage.java | 291 +++++++++++++++++++
.../apache/cassandra/hadoop/pig/CqlStorage.java | 11 +-
.../cassandra/metrics/KeyspaceMetrics.java | 2 +-
.../apache/cassandra/service/ClientState.java | 3 +-
.../org/apache/cassandra/tools/NodeTool.java | 4 +-
.../apache/cassandra/transport/DataType.java | 3 +-
test/conf/cassandra.yaml | 2 +
.../cassandra/pig/CqlTableDataTypeTest.java | 94 +++++-
.../org/apache/cassandra/pig/CqlTableTest.java | 101 ++++++-
.../org/apache/cassandra/pig/PigTestBase.java | 3 +
.../cassandra/pig/ThriftColumnFamilyTest.java | 53 +++-
.../apache/cassandra/stress/StressAction.java | 22 +-
.../apache/cassandra/stress/StressMetrics.java | 3 +-
.../apache/cassandra/stress/StressProfile.java | 2 +-
.../stress/settings/SettingsCommand.java | 76 ++++-
.../settings/SettingsCommandPreDefined.java | 6 +-
.../SettingsCommandPreDefinedMixed.java | 6 +-
.../stress/settings/SettingsCommandUser.java | 3 +-
34 files changed, 692 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/432cfeff/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ae1940f,d4dd5d9..9c122f9
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -22,11 -24,14 +24,15 @@@
* Optimise NativeCell comparisons (CASSANDRA-6755)
* Configurable client timeout for cqlsh (CASSANDRA-7516)
* Include snippet of CQL query near syntax error in messages (CASSANDRA-7111)
+Merged from 2.0:
+ * Catch errors when the JVM pulls the rug out from GCInspector (CASSANDRA-5345)
+ * Fix native protocol drop user type notification (CASSANDRA-7571)
+ * Give read access to system.schema_usertypes to all authenticated users
+ (CASSANDRA-7578)
-Merged from 2.0:
+ * Fix ReversedType(DateType) mapping to native protocol (CASSANDRA-7576)
- 2.1.0-final
+ 2.1.0-rc4
* Updated memtable_cleanup_threshold and memtable_flush_writers defaults
(CASSANDRA-7551)
* (Windows) fix startup when WMI memory query fails (CASSANDRA-7505)
[7/7] git commit: merge
Posted by jb...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f7e3b9a6
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f7e3b9a6
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f7e3b9a6
Branch: refs/heads/cassandra-2.1
Commit: f7e3b9a6b310924b11e09a7cc37fc83a5eb2b948
Parents: b181261 f9fc846
Author: Jonathan Ellis <jb...@apache.org>
Authored: Wed Jul 23 12:55:19 2014 -0700
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Wed Jul 23 12:56:00 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 7 +-
bin/cqlsh | 3 +-
build.xml | 2 +-
conf/cassandra-env.sh | 4 +-
debian/changelog | 6 +
doc/cql/CQL.textile | 2 +
doc/cql3/CQL.textile | 2 +-
examples/hadoop_cql3_word_count/README.txt | 6 +-
.../conf/log4j.properties | 2 +-
.../src/WordCountSetup.java | 5 +-
examples/hadoop_word_count/README.txt | 6 +-
.../hadoop_word_count/conf/log4j.properties | 2 +-
examples/hadoop_word_count/src/WordCount.java | 5 +-
.../src/WordCountCounters.java | 2 +
.../hadoop_word_count/src/WordCountSetup.java | 25 +-
pylib/cqlshlib/cql3handling.py | 5 +-
.../org/apache/cassandra/db/DefsTables.java | 2 +-
.../cassandra/hadoop/cql3/CqlConfigHelper.java | 1 -
.../cassandra/hadoop/cql3/CqlInputFormat.java | 2 -
.../cassandra/hadoop/pig/CqlNativeStorage.java | 291 +++++++++++++++++++
.../apache/cassandra/hadoop/pig/CqlStorage.java | 11 +-
.../apache/cassandra/service/ClientState.java | 3 +-
.../org/apache/cassandra/tools/NodeTool.java | 4 +-
.../apache/cassandra/transport/DataType.java | 3 +-
test/conf/cassandra.yaml | 2 +
.../cassandra/pig/CqlTableDataTypeTest.java | 94 +++++-
.../org/apache/cassandra/pig/CqlTableTest.java | 101 ++++++-
.../org/apache/cassandra/pig/PigTestBase.java | 3 +
.../cassandra/pig/ThriftColumnFamilyTest.java | 53 +++-
29 files changed, 590 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/f7e3b9a6/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index a30de32,6914a84..4432b10
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -6,11 -7,14 +7,15 @@@
* Optimise NativeCell comparisons (CASSANDRA-6755)
* Configurable client timeout for cqlsh (CASSANDRA-7516)
* Include snippet of CQL query near syntax error in messages (CASSANDRA-7111)
+Merged from 2.0:
+ * Catch errors when the JVM pulls the rug out from GCInspector (CASSANDRA-5345)
+ * Fix native protocol drop user type notification (CASSANDRA-7571)
+ * Give read access to system.schema_usertypes to all authenticated users
+ (CASSANDRA-7578)
-Merged from 2.0:
+ * Fix ReversedType(DateType) mapping to native protocol (CASSANDRA-7576)
- 2.1.0-final
+ 2.1.0-rc4
* Updated memtable_cleanup_threshold and memtable_flush_writers defaults
(CASSANDRA-7551)
* (Windows) fix startup when WMI memory query fails (CASSANDRA-7505)
[2/7] git commit: Catch errors when the JVM pulls the rug out from
GCInspector patch by Josh McKenzie; reviewed by jbellis for CASSANDRA-5345
Posted by jb...@apache.org.
Catch errors when the JVM pulls the rug out from GCInspector
patch by Josh McKenzie; reviewed by jbellis for CASSANDRA-5345
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/51a1f232
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/51a1f232
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/51a1f232
Branch: refs/heads/trunk
Commit: 51a1f232b42b4385df7f2aca42a97b4c5b02c077
Parents: 062addb
Author: Jonathan Ellis <jb...@apache.org>
Authored: Fri Jul 18 18:13:33 2014 -0500
Committer: Jonathan Ellis <jb...@apache.org>
Committed: Fri Jul 18 18:13:33 2014 -0500
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/service/GCInspector.java | 130 +++++++++++--------
2 files changed, 79 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/51a1f232/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d339309..d4b11d1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
2.0.10
+ * Catch errors when the JVM pulls the rug out from GCInspector (CASSANDRA-5345)
* (Windows) force range-based repair to non-sequential mode (CASSANDRA-7541)
* Fix range merging when DES scores are zero (CASSANDRA-7535)
* Warn when SSL certificates have expired (CASSANDRA-7528)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/51a1f232/src/java/org/apache/cassandra/service/GCInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java
index 9961bf9..f03ec01 100644
--- a/src/java/org/apache/cassandra/service/GCInspector.java
+++ b/src/java/org/apache/cassandra/service/GCInspector.java
@@ -21,6 +21,7 @@ import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
+import java.lang.reflect.UndeclaredThrowableException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -49,8 +50,28 @@ public class GCInspector
final List<GarbageCollectorMXBean> beans = new ArrayList<GarbageCollectorMXBean>();
final MemoryMXBean membean = ManagementFactory.getMemoryMXBean();
- public GCInspector()
+ public void start()
{
+ buildMXBeanList();
+
+ // don't bother starting a thread that will do nothing.
+ if (beans.isEmpty())
+ return;
+
+ Runnable t = new Runnable()
+ {
+ public void run()
+ {
+ logGCResults();
+ }
+ };
+ StorageService.scheduledTasks.scheduleWithFixedDelay(t, INTERVAL_IN_MS, INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
+ }
+
+ private void buildMXBeanList()
+ {
+ beans.clear();
+
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
try
{
@@ -67,62 +88,67 @@ public class GCInspector
}
}
- public void start()
+ private void logGCResults()
{
- // don't bother starting a thread that will do nothing.
- if (beans.size() == 0)
- return;
- Runnable t = new Runnable()
+ boolean gcChanged = false;
+ try
{
- public void run()
+ for (GarbageCollectorMXBean gc : beans)
{
- logGCResults();
+ if (!gc.isValid())
+ {
+ gcChanged = true;
+ continue;
+ }
+
+ Long previousTotal = gctimes.get(gc.getName());
+ Long total = gc.getCollectionTime();
+ if (previousTotal == null)
+ previousTotal = 0L;
+ if (previousTotal.equals(total))
+ continue;
+ gctimes.put(gc.getName(), total);
+ Long duration = total - previousTotal; // may be zero for a really fast collection
+
+ Long previousCount = gccounts.get(gc.getName());
+ Long count = gc.getCollectionCount();
+
+ if (previousCount == null)
+ previousCount = 0L;
+ if (count.equals(previousCount))
+ continue;
+
+ gccounts.put(gc.getName(), count);
+
+ MemoryUsage mu = membean.getHeapMemoryUsage();
+ long memoryUsed = mu.getUsed();
+ long memoryMax = mu.getMax();
+
+ String st = String.format("GC for %s: %s ms for %s collections, %s used; max is %s",
+ gc.getName(), duration, count - previousCount, memoryUsed, memoryMax);
+ long durationPerCollection = duration / (count - previousCount);
+ if (durationPerCollection > MIN_DURATION)
+ logger.info(st);
+ else if (logger.isDebugEnabled())
+ logger.debug(st);
+
+ if (durationPerCollection > MIN_DURATION_TPSTATS)
+ StatusLogger.log();
+
+ // if we just finished a full collection and we're still using a lot of memory, try to reduce the pressure
+ if (gc.getName().equals("ConcurrentMarkSweep"))
+ SSTableDeletingTask.rescheduleFailedTasks();
}
- };
- StorageService.scheduledTasks.scheduleWithFixedDelay(t, INTERVAL_IN_MS, INTERVAL_IN_MS, TimeUnit.MILLISECONDS);
- }
-
- private void logGCResults()
- {
- for (GarbageCollectorMXBean gc : beans)
+ }
+ catch (UndeclaredThrowableException e)
{
- Long previousTotal = gctimes.get(gc.getName());
- Long total = gc.getCollectionTime();
- if (previousTotal == null)
- previousTotal = 0L;
- if (previousTotal.equals(total))
- continue;
- gctimes.put(gc.getName(), total);
- Long duration = total - previousTotal; // may be zero for a really fast collection
-
- Long previousCount = gccounts.get(gc.getName());
- Long count = gc.getCollectionCount();
-
- if (previousCount == null)
- previousCount = 0L;
- if (count.equals(previousCount))
- continue;
-
- gccounts.put(gc.getName(), count);
-
- MemoryUsage mu = membean.getHeapMemoryUsage();
- long memoryUsed = mu.getUsed();
- long memoryMax = mu.getMax();
-
- String st = String.format("GC for %s: %s ms for %s collections, %s used; max is %s",
- gc.getName(), duration, count - previousCount, memoryUsed, memoryMax);
- long durationPerCollection = duration / (count - previousCount);
- if (durationPerCollection > MIN_DURATION)
- logger.info(st);
- else if (logger.isDebugEnabled())
- logger.debug(st);
-
- if (durationPerCollection > MIN_DURATION_TPSTATS)
- StatusLogger.log();
-
- // if we just finished a full collection and we're still using a lot of memory, try to reduce the pressure
- if (gc.getName().equals("ConcurrentMarkSweep"))
- SSTableDeletingTask.rescheduleFailedTasks();
+ // valid-ness may have changed out from under us, even though we check for it explicitly.
+ // if so, gc.getName() will throw UTE when reflection runs into InstanceNotFoundException.
+ // See CASSANDRA-5345
+ gcChanged = true;
}
+
+ if (gcChanged)
+ buildMXBeanList();
}
}