You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2016/03/14 19:23:57 UTC
tez git commit: TEZ-3105. TezMxBeanResourceCalculator does not work
on IBM JDK 7 or 8 causing Tez failures. (Greg Senia via hitesh)
Repository: tez
Updated Branches:
refs/heads/branch-0.7 1cb6e0f82 -> 37ae6f561
TEZ-3105. TezMxBeanResourceCalculator does not work on IBM JDK 7 or 8 causing Tez failures. (Greg Senia via hitesh)
(cherry picked from commit fac2f5fe1e84ed57db045d0467529b3627b91458)
Conflicts:
CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/37ae6f56
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/37ae6f56
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/37ae6f56
Branch: refs/heads/branch-0.7
Commit: 37ae6f561e2ca9310f7ab731fd4e9dce0601e8f6
Parents: 1cb6e0f
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Mar 14 11:17:51 2016 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Mar 14 11:22:31 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../tez/util/TezMxBeanResourceCalculator.java | 65 ++++++++++++++++++--
2 files changed, 60 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/37ae6f56/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c0870a5..bb7a56d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-2972. Avoid task rescheduling when a node turns unhealthy
ALL CHANGES:
+ TEZ-3105. TezMxBeanResourceCalculator does not work on IBM JDK 7 or 8 causing Tez failures.
TEZ-2863. Container, node, and logs not available in UI for tasks that fail to launch
TEZ-3140. Reduce AM memory usage during serialization
TEZ-3156. Tez client keeps trying to talk to RM even if RM does not know about the application.
http://git-wip-us.apache.org/repos/asf/tez/blob/37ae6f56/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
index 1f8e922..07f34e6 100644
--- a/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
+++ b/tez-common/src/main/java/org/apache/tez/util/TezMxBeanResourceCalculator.java
@@ -22,14 +22,19 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import java.lang.management.ManagementFactory;
import java.util.concurrent.TimeUnit;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
/**
- * Uses sun's MBeans to return process information.
+ * Uses Sun/Oracle or IBM MBeans to return process information.
*/
public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
- private final com.sun.management.OperatingSystemMXBean osBean;
+ private final OperatingSystemMXBean osBean;
private final Runtime runtime;
+ private static final Method getCommittedVirtualMemorySize = getMxBeanMethod("getCommittedVirtualMemorySize");
+ private static final Method getProcessCpuTime = getMxBeanMethod("getProcessCpuTime");
/**
* Create process-tree instance with specified root process.
@@ -41,8 +46,8 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
public TezMxBeanResourceCalculator(String root) {
super(root);
runtime = Runtime.getRuntime();
- osBean =
- (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+ osBean = ManagementFactory.getOperatingSystemMXBean();
+
}
@Override public void updateProcessTree() {
@@ -54,7 +59,15 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
}
@Override public long getCumulativeVmem(int olderThanAge) {
- return osBean.getCommittedVirtualMemorySize();
+ try {
+ return (Long) getCommittedVirtualMemorySize.invoke(osBean);
+ } catch (IllegalArgumentException e) {
+ return -1;
+ } catch (IllegalAccessException e) {
+ return -1;
+ } catch (InvocationTargetException e) {
+ return -1;
+ }
}
@Override public long getCumulativeRssmem(int olderThanAge) {
@@ -64,7 +77,16 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
@Override public long getCumulativeCpuTime() {
//convert to milliseconds
- return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS);
+ try {
+ return TimeUnit.MILLISECONDS.convert(
+ (Long) getProcessCpuTime.invoke(osBean), TimeUnit.NANOSECONDS);
+ } catch (InvocationTargetException e) {
+ return -1;
+ } catch (IllegalArgumentException e) {
+ return -1;
+ } catch (IllegalAccessException e) {
+ return -1;
+ }
}
@Override public boolean checkPidPgrpidForMatch() {
@@ -76,4 +98,35 @@ public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
//Returning -1 to indicate, this feature is not yet supported.
return -1;
}
+
+ private static Method getMxBeanMethod(String methodName) {
+ // New Method to support IBM and Oracle/OpenJDK JDK with OperatingSystemMXBean
+ final String JAVA_VENDOR_NAME = System.getProperty("java.vendor");
+ final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM");
+ try {
+ final Class<?> mbeanClazz;
+ if (IBM_JAVA) {
+ mbeanClazz = Class.forName("com.ibm.lang.management.OperatingSystemMXBean");
+ } else {
+ mbeanClazz = Class.forName("com.sun.management.OperatingSystemMXBean");
+ }
+ if (IBM_JAVA){
+ if (methodName.equals("getCommittedVirtualMemorySize")) {
+ methodName = "getProcessVirtualMemorySize";
+ }
+ if (methodName.equals("getProcessCpuTime")) {
+ methodName = "getProcessCpuTimeByNS";
+ }
+ }
+ final Method method = mbeanClazz
+ .getMethod(methodName);
+ return method;
+ } catch (ClassNotFoundException e) {
+ return null;
+ } catch (SecurityException e) {
+ return null;
+ } catch (NoSuchMethodException e) {
+ return null;
+ }
+ }
}