You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@tez.apache.org by "Greg Senia (JIRA)" <ji...@apache.org> on 2016/02/11 00:54:18 UTC
[jira] [Comment Edited] (TEZ-3105) Tez does not run on IBM JDK 7 or
8
[ https://issues.apache.org/jira/browse/TEZ-3105?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15141972#comment-15141972 ]
Greg Senia edited comment on TEZ-3105 at 2/10/16 11:53 PM:
-----------------------------------------------------------
[~gopalv] second run of the fix that generated the NPE....
{Code}
public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
private final OperatingSystemMXBean osBean;
private final Runtime runtime;
private static final String JAVA_VENDOR_NAME = System.getProperty("java.vendor");
private static final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM");
private final Class<?> mbeanClazz = getMBeanClass();
private final Method vmemMethod = getVmemMethod();
private final Method cpuMethod = getProcCpuMethod();
/**
* Create process-tree instance with specified root process.
* <p/>
* Subclass must override this.
*
* @param root process-tree root-process
*/
public TezMxBeanResourceCalculator(String root) {
super(root);
runtime = Runtime.getRuntime();
osBean =
ManagementFactory.getOperatingSystemMXBean();
}
@Override public void updateProcessTree() {
//nothing needs to be done as the data is read from OS mbeans.
}
@Override public String getProcessTreeDump() {
return "";
}
@Override public long getCumulativeVmem(int olderThanAge) {
long vmem = 0L;
try {
vmem= (Long) vmemMethod.invoke(osBean);
} catch ( IllegalAccessException iae ) {
} catch ( IllegalArgumentException iare ) {
} catch ( InvocationTargetException ite ) {
} catch ( SecurityException se ) {
}
return vmem;
}
@Override public long getCumulativeRssmem(int olderThanAge) {
//Not supported directly (RSS ~= memory consumed by JVM from Xmx)
return runtime.totalMemory();
}
@Override public long getCumulativeCpuTime() {
//convert to milliseconds
long cputime = 0L;
try {
cputime= TimeUnit.MILLISECONDS.convert((Long) cpuMethod.invoke(osBean), TimeUnit.NANOSECONDS);
} catch ( IllegalAccessException iae ) {
} catch ( IllegalArgumentException iare ) {
} catch ( InvocationTargetException ite ) {
} catch ( SecurityException se ) {
}
return cputime;
}
@Override public boolean checkPidPgrpidForMatch() {
return true;
}
public float getCpuUsagePercent() {
//osBean.getProcessCpuLoad() can be closer and returns [0 - 1.0], but might not be accurate.
//Returning -1 to indicate, this feature is not yet supported.
return -1;
}
public final Class<?> getMBeanClass() {
try {
if (IBM_JAVA) {
return Class.forName("com.ibm.management.OperatingSystemMXBean");
} else {
return Class.forName("com.sun.management.OperatingSystemMXBean");
}
} catch ( Exception e ) {
e.printStackTrace();
return null;
}
}
public final Method getVmemMethod() {
try {
return mbeanClazz.getMethod("getCommittedVirtualMemorySize");
} catch ( Exception e ) {
e.printStackTrace();
return null;
}
}
public final Method getProcCpuMethod() {
try {
return mbeanClazz.getMethod("getProcessCpuTime");
} catch ( Exception e ) {
e.printStackTrace();
return null;
}
}
}
{code}
was (Author: gss2002):
[~gopalv] second run of the fix that generated the NPE....
{Code}
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
import java.util.concurrent.TimeUnit;
/**
- * Uses sun's MBeans to return process information.
+ * Uses Sun or IBMs MBeans to return process information.
*/
public class TezMxBeanResourceCalculator extends ResourceCalculatorProcessTree {
- private final com.sun.management.OperatingSystemMXBean osBean;
+ private final OperatingSystemMXBean osBean;
private final Runtime runtime;
+ private static final String JAVA_VENDOR_NAME = System.getProperty("java.vendor");
+ private static final boolean IBM_JAVA = JAVA_VENDOR_NAME.contains("IBM");
+ private final Class<?> mbeanClazz = getMBeanClass();
+ private final Method vmemMethod = getVmemMethod();
+ private final Method cpuMethod = getProcCpuMethod();
/**
* Create process-tree instance with specified root process.
@@ -42,7 +50,7 @@
super(root);
runtime = Runtime.getRuntime();
osBean =
- (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
+ ManagementFactory.getOperatingSystemMXBean();
}
@Override public void updateProcessTree() {
@@ -54,7 +62,16 @@
}
@Override public long getCumulativeVmem(int olderThanAge) {
- return osBean.getCommittedVirtualMemorySize();
+ long vmem = 0L;
+ try {
+ vmem= (Long) vmemMethod.invoke(osBean);
+ } catch ( IllegalAccessException iae ) {
+ } catch ( IllegalArgumentException iare ) {
+ } catch ( InvocationTargetException ite ) {
+ } catch ( SecurityException se ) {
+ }
+ return vmem;
+
}
@Override public long getCumulativeRssmem(int olderThanAge) {
@@ -64,7 +81,15 @@
@Override public long getCumulativeCpuTime() {
//convert to milliseconds
- return TimeUnit.MILLISECONDS.convert(osBean.getProcessCpuTime(), TimeUnit.NANOSECONDS);
+ long cputime = 0L;
+ try {
+ cputime= TimeUnit.MILLISECONDS.convert((Long) cpuMethod.invoke(osBean), TimeUnit.NANOSECONDS);
+ } catch ( IllegalAccessException iae ) {
+ } catch ( IllegalArgumentException iare ) {
+ } catch ( InvocationTargetException ite ) {
+ } catch ( SecurityException se ) {
+ }
+ return cputime;
}
@Override public boolean checkPidPgrpidForMatch() {
@@ -76,4 +101,35 @@
//Returning -1 to indicate, this feature is not yet supported.
return -1;
}
+
+ public final Class<?> getMBeanClass() {
+ try {
+ if (IBM_JAVA) {
+ return Class.forName("com.ibm.management.OperatingSystemMXBean");
+ } else {
+ return Class.forName("com.sun.management.OperatingSystemMXBean");
+ }
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ public final Method getVmemMethod() {
+ try {
+ return mbeanClazz.getMethod("getCommittedVirtualMemorySize");
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ public final Method getProcCpuMethod() {
+ try {
+ return mbeanClazz.getMethod("getProcessCpuTime");
+ } catch ( Exception e ) {
+ e.printStackTrace();
+ return null;
+ }
+ }
{code}
> Tez does not run on IBM JDK 7 or 8
> ----------------------------------
>
> Key: TEZ-3105
> URL: https://issues.apache.org/jira/browse/TEZ-3105
> Project: Apache Tez
> Issue Type: Bug
> Affects Versions: 0.7.0
> Reporter: Greg Senia
> Assignee: Greg Senia
> Labels: ibm, ibm-jdk
> Attachments: TEZ-3105-2.patch, TEZ-3105.patch
>
>
> When testing Hive on Tez with IBM JDK 7 and 8. The following issue was discovered:
> 2016-02-08 22:25:22,869 [ERROR] [main] |app.DAGAppMaster|: Error starting DAGAppMaster
> java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
> at org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(ResourceCalculatorProcessTree.java:225)
> at org.apache.tez.dag.app.DAGAppMaster.initResourceCalculatorPlugins(DAGAppMaster.java:347)
> at org.apache.tez.dag.app.DAGAppMaster.serviceInit(DAGAppMaster.java:371)
> at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
> at org.apache.tez.dag.app.DAGAppMaster$6.run(DAGAppMaster.java:2274)
> at java.security.AccessController.doPrivileged(AccessController.java:686)
> at javax.security.auth.Subject.doAs(Subject.java:569)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
> at org.apache.tez.dag.app.DAGAppMaster.initAndStartAppMaster(DAGAppMaster.java:2271)
> at org.apache.tez.dag.app.DAGAppMaster.main(DAGAppMaster.java:2086)
> Caused by: java.lang.reflect.InvocationTargetException
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:88)
> at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:57)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:436)
> at org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(ResourceCalculatorProcessTree.java:221)
> ... 9 more
> Caused by: java.lang.ClassCastException: com.ibm.lang.management.ExtendedOperatingSystem incompatible with com.sun.management.OperatingSystemMXBean
> at org.apache.tez.util.TezMxBeanResourceCalculator.<init>(TezMxBeanResourceCalculator.java:44)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)