You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2014/06/24 13:46:12 UTC
git commit: Add garbage collector stats to output of TaskManager
memory usage debug thread
Repository: incubator-flink
Updated Branches:
refs/heads/master 7d0e89d2b -> 1357ea3a4
Add garbage collector stats to output of TaskManager memory usage debug thread
Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/1357ea3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/1357ea3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/1357ea3a
Branch: refs/heads/master
Commit: 1357ea3a40f747e4e51b5b52612323fe55747da7
Parents: 7d0e89d
Author: uce <u....@fu-berlin.de>
Authored: Tue Jun 24 13:43:13 2014 +0200
Committer: uce <u....@fu-berlin.de>
Committed: Tue Jun 24 13:43:13 2014 +0200
----------------------------------------------------------------------
.../nephele/taskmanager/TaskManager.java | 35 ++++++++++++++++----
.../exampleJavaPrograms/WordCountITCase.java | 4 +++
2 files changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1357ea3a/stratosphere-runtime/src/main/java/eu/stratosphere/nephele/taskmanager/TaskManager.java
----------------------------------------------------------------------
diff --git a/stratosphere-runtime/src/main/java/eu/stratosphere/nephele/taskmanager/TaskManager.java b/stratosphere-runtime/src/main/java/eu/stratosphere/nephele/taskmanager/TaskManager.java
index 9b623bd..789955a 100644
--- a/stratosphere-runtime/src/main/java/eu/stratosphere/nephele/taskmanager/TaskManager.java
+++ b/stratosphere-runtime/src/main/java/eu/stratosphere/nephele/taskmanager/TaskManager.java
@@ -15,6 +15,7 @@ package eu.stratosphere.nephele.taskmanager;
import java.io.File;
import java.io.IOException;
+import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
@@ -408,9 +409,10 @@ public class TaskManager implements TaskOperationProtocol {
// Memory Usage
// --------------------------------------------------------------------
- final MemoryMXBean memoryUsageBean = ManagementFactory.getMemoryMXBean();
+ final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+ final List<GarbageCollectorMXBean> gcMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
- LOG.info(getMemoryUsageAsString(memoryUsageBean));
+ LOG.info(getMemoryUsageStatsAsString(memoryMXBean));
boolean startMemoryUsageLogThread = GlobalConfiguration.getBoolean(
ConfigConstants.TASK_MANAGER_DEBUG_MEMORY_USAGE_START_LOG_THREAD,
@@ -427,7 +429,10 @@ public class TaskManager implements TaskOperationProtocol {
try {
while (!isShutDown()) {
Thread.sleep(logIntervalMs);
- LOG.debug(getMemoryUsageAsString(memoryUsageBean));
+
+ LOG.debug(getMemoryUsageStatsAsString(memoryMXBean));
+
+ LOG.debug(getGarbageCollectorStatsAsString(gcMXBeans));
}
} catch (InterruptedException e) {
LOG.warn("Unexpected interruption of memory usage logger thread.");
@@ -1120,9 +1125,9 @@ public class TaskManager implements TaskOperationProtocol {
}
}
- private String getMemoryUsageAsString(MemoryMXBean memoryUsageBean) {
- MemoryUsage heap = memoryUsageBean.getHeapMemoryUsage();
- MemoryUsage nonHeap = memoryUsageBean.getNonHeapMemoryUsage();
+ private String getMemoryUsageStatsAsString(MemoryMXBean memoryMXBean) {
+ MemoryUsage heap = memoryMXBean.getHeapMemoryUsage();
+ MemoryUsage nonHeap = memoryMXBean.getNonHeapMemoryUsage();
int mb = 1 << 20;
@@ -1134,9 +1139,25 @@ public class TaskManager implements TaskOperationProtocol {
int nonHeapCommitted = (int) (nonHeap.getCommitted() / mb);
int nonHeapMax = (int) (nonHeap.getMax() / mb);
- String msg = String.format("Memory usage HEAP: %d/%d/%d MB, NON HEAP: %d/%d/%d MB (used/comitted/max)",
+ String msg = String.format("Memory usage stats: [HEAP: %d/%d/%d MB, NON HEAP: %d/%d/%d MB (used/comitted/max)]",
heapUsed, heapCommitted, heapMax, nonHeapUsed, nonHeapCommitted, nonHeapMax);
return msg;
}
+
+ private String getGarbageCollectorStatsAsString(List<GarbageCollectorMXBean> gcMXBeans) {
+ StringBuilder str = new StringBuilder();
+ str.append("Garbage collector stats: ");
+
+ for (int i = 0; i < gcMXBeans.size(); i++) {
+ GarbageCollectorMXBean bean = gcMXBeans.get(i);
+
+ String msg = String.format("[%s, GC TIME (ms): %d, GC COUNT: %d]",
+ bean.getName(), bean.getCollectionTime(), bean.getCollectionCount());
+ str.append(msg);
+ str.append(i < gcMXBeans.size() - 1 ? ", " : "");
+ }
+
+ return str.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/1357ea3a/stratosphere-tests/src/test/java/eu/stratosphere/test/exampleJavaPrograms/WordCountITCase.java
----------------------------------------------------------------------
diff --git a/stratosphere-tests/src/test/java/eu/stratosphere/test/exampleJavaPrograms/WordCountITCase.java b/stratosphere-tests/src/test/java/eu/stratosphere/test/exampleJavaPrograms/WordCountITCase.java
index 272bce6..ec67390 100644
--- a/stratosphere-tests/src/test/java/eu/stratosphere/test/exampleJavaPrograms/WordCountITCase.java
+++ b/stratosphere-tests/src/test/java/eu/stratosphere/test/exampleJavaPrograms/WordCountITCase.java
@@ -17,6 +17,8 @@ package eu.stratosphere.test.exampleJavaPrograms;
import eu.stratosphere.example.java.wordcount.WordCount;
import eu.stratosphere.test.testdata.WordCountData;
import eu.stratosphere.test.util.JavaProgramTestBase;
+import eu.stratosphere.util.LogUtils;
+import org.apache.log4j.Level;
public class WordCountITCase extends JavaProgramTestBase {
@@ -25,6 +27,8 @@ public class WordCountITCase extends JavaProgramTestBase {
protected String resultPath;
public WordCountITCase(){
+ LogUtils.initializeDefaultConsoleLogger(Level.DEBUG);
+
setDegreeOfParallelism(4);
setNumTaskTracker(2);
setTaskManagerNumSlots(2);