You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@tez.apache.org by "r7raul1984@163.com" <r7...@163.com> on 2015/06/04 07:19:12 UTC
hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
hive> set hive.execution.engine=tez;
hive> set hive.tez.exec.print.summary=true;
hive> select userid,count(*) from u_data group by userid order by userid;
Query ID = hdfs_20150604131717_6863a8c8-c3a6-4865-b098-06aff509dbf8
Total jobs = 1
Launching Job 1 out of 1
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(ATSHistoryACLPolicyManager.java:120)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createSessionDomain(ATSHistoryACLPolicyManager.java:167)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionACLs(ATSHistoryACLPolicyManager.java:228)
at org.apache.tez.client.TezClientUtils.createApplicationSubmissionContext(TezClientUtils.java:525)
at org.apache.tez.client.TezClient.start(TezClient.java:352)
at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:191)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.updateSession(TezTask.java:234)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:136)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160)
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:88)
at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1638)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1397)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1183)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1039)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:207)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:159)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:370)
at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:754)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.main(RunJar.java:212)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.api.records.timeline.TimelineDomain
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 26 more
FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.tez.TezTask. org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
hive>
r7raul1984@163.com
Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
Posted by "Jianfeng (Jeff) Zhang" <jz...@hortonworks.com>.
Hadoop 2.5.0 don't have Timeline with ACLs Support, please set the following property in tez-site.xml
<property>
<name>tez.allow.disabled.timeline-domains</name>
<value>true</value>
</property>
For more details, please check this link
http://tez.apache.org/tez_yarn_timeline.html
Best Regard,
Jeff Zhang
From: "r7raul1984@163.com<ma...@163.com>" <r7...@163.com>>
Reply-To: user <us...@tez.apache.org>>
Date: Thursday, June 4, 2015 at 1:19 PM
To: user <us...@tez.apache.org>>
Subject: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
hive> set hive.execution.engine=tez;
hive> set hive.tez.exec.print.summary=true;
hive> select userid,count(*) from u_data group by userid order by userid;
Query ID = hdfs_20150604131717_6863a8c8-c3a6-4865-b098-06aff509dbf8
Total jobs = 1
Launching Job 1 out of 1
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(ATSHistoryACLPolicyManager.java:120)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createSessionDomain(ATSHistoryACLPolicyManager.java:167)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionACLs(ATSHistoryACLPolicyManager.java:228)
at org.apache.tez.client.TezClientUtils.createApplicationSubmissionContext(TezClientUtils.java:525)
at org.apache.tez.client.TezClient.start(TezClient.java:352)
at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:191)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.updateSession(TezTask.java:234)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:136)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160)
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:88)
at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1638)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1397)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1183)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1039)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:207)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:159)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:370)
at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:754)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.main(RunJar.java:212)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.api.records.timeline.TimelineDomain
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 26 more
FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.tez.TezTask. org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
hive>
________________________________
r7raul1984@163.com<ma...@163.com>
Re: Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
Posted by "r7raul1984@163.com" <r7...@163.com>.
ok I will try . My hadoop version is
Hadoop 2.5.0-cdh5.2.0
Subversion http://github.com/cloudera/hadoop -r e1f20a08bde76a33b79df026d00a0c91b2298387
Compiled by jenkins on 2014-10-11T21:00Z
Compiled with protoc 2.5.0
From source with checksum 309bccd135b199bdfdd6df5f3f4153d
This command was run using /opt/cloudera/parcels/CDH-5.2.0-1.cdh5.2.0.p0.36/jars/hadoop-common-2.5.0-cdh5.2.0.jar
r7raul1984@163.com
From: Jianfeng (Jeff) Zhang
Date: 2015-06-04 14:00
To: user
Subject: Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
I believe you are using CDH. I remember there’s one thread in hive mail list also mention this issue.
If I remember it correctly, you need to rebuild tez with CDH hadoop version.
Best Regard,
Jeff Zhang
From: "r7raul1984@163.com" <r7...@163.com>
Reply-To: user <us...@tez.apache.org>
Date: Thursday, June 4, 2015 at 1:54 PM
To: user <us...@tez.apache.org>
Subject: Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
I commet
<!-- <property>
<description>Log history using the Timeline Server</description>
<name>tez.history.logging.service.class</name>
<value>org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService</value>
</property>
<property>
<description>Publish configuration information to Timeline server.-XX:+UseParallelGC -XX:+UseG1GC </description>
<name>tez.runtime.convert.user-payload.to.history-text</name>
<value>true</value>
</property>-->
The problem not occur.
Then I try run hadoop jar tez-examples-0.7.0.jar orderedwordcount /usr/tez/tez-site.xml /tmp . Return error:
Stack trace: ExitCodeException exitCode=1:
at org.apache.hadoop.util.Shell.runCommand(Shell.java:538)
at org.apache.hadoop.util.Shell.run(Shell.java:455)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:702)
at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:196)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:299)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:81)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
I check log for detail ,I found:
2015-06-04 13:46:04,218 INFO [main] app.DAGAppMaster: Running DAG: OrderedWordCount
2015-06-04 13:46:04,263 ERROR [main] app.DAGAppMaster: Error starting DAGAppMaster
java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V
at org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(Native Method)
at org.apache.hadoop.util.NativeCrc32.calculateChunkedSumsByteArray(NativeCrc32.java:86)
at org.apache.hadoop.util.DataChecksum.calculateChunkedSums(DataChecksum.java:430)
at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:202)
at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:163)
at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1993)
at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1946)
at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
at org.apache.tez.dag.history.recovery.RecoveryService.handle(RecoveryService.java:286)
at org.apache.tez.dag.history.HistoryEventHandler.handleCriticalEvent(HistoryEventHandler.java:102)
at org.apache.tez.dag.app.DAGAppMaster.startDAG(DAGAppMaster.java:2153)
at org.apache.tez.dag.app.DAGAppMaster.startDAG(DAGAppMaster.java:2103)
at org.apache.tez.dag.app.DAGAppMaster.serviceStart(DAGAppMaster.java:1798)
at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193)
at org.apache.tez.dag.app.DAGAppMaster$6.run(DAGAppMaster.java:2224)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
at org.apache.tez.dag.app.DAGAppMaster.initAndStartAppMaster(DAGAppMaster.java:2220)
at org.apache.tez.dag.app.DAGAppMaster.main(DAGAppMaster.java:2038)
r7raul1984@163.com
From: r7raul1984@163.com
Date: 2015-06-04 13:19
To: user
Subject: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
hive> set hive.execution.engine=tez;
hive> set hive.tez.exec.print.summary=true;
hive> select userid,count(*) from u_data group by userid order by userid;
Query ID = hdfs_20150604131717_6863a8c8-c3a6-4865-b098-06aff509dbf8
Total jobs = 1
Launching Job 1 out of 1
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(ATSHistoryACLPolicyManager.java:120)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createSessionDomain(ATSHistoryACLPolicyManager.java:167)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionACLs(ATSHistoryACLPolicyManager.java:228)
at org.apache.tez.client.TezClientUtils.createApplicationSubmissionContext(TezClientUtils.java:525)
at org.apache.tez.client.TezClient.start(TezClient.java:352)
at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:191)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.updateSession(TezTask.java:234)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:136)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160)
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:88)
at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1638)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1397)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1183)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1039)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:207)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:159)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:370)
at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:754)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.main(RunJar.java:212)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.api.records.timeline.TimelineDomain
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 26 more
FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.tez.TezTask. org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
hive>
r7raul1984@163.com
Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
Posted by "Jianfeng (Jeff) Zhang" <jz...@hortonworks.com>.
I believe you are using CDH. I remember there’s one thread in hive mail list also mention this issue.
If I remember it correctly, you need to rebuild tez with CDH hadoop version.
Best Regard,
Jeff Zhang
From: "r7raul1984@163.com<ma...@163.com>" <r7...@163.com>>
Reply-To: user <us...@tez.apache.org>>
Date: Thursday, June 4, 2015 at 1:54 PM
To: user <us...@tez.apache.org>>
Subject: Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
I commet
<!-- <property>
<description>Log history using the Timeline Server</description>
<name>tez.history.logging.service.class</name>
<value>org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService</value>
</property>
<property>
<description>Publish configuration information to Timeline server.-XX:+UseParallelGC -XX:+UseG1GC </description>
<name>tez.runtime.convert.user-payload.to.history-text</name>
<value>true</value>
</property>-->
The problem not occur.
Then I try run hadoop jar tez-examples-0.7.0.jar orderedwordcount /usr/tez/tez-site.xml /tmp . Return error:
Stack trace: ExitCodeException exitCode=1:
at org.apache.hadoop.util.Shell.runCommand(Shell.java:538)
at org.apache.hadoop.util.Shell.run(Shell.java:455)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:702)
at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:196)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:299)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:81)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
I check log for detail ,I found:
2015-06-04 13:46:04,218 INFO [main] app.DAGAppMaster: Running DAG: OrderedWordCount
2015-06-04 13:46:04,263 ERROR [main] app.DAGAppMaster: Error starting DAGAppMaster
java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V
at org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(Native Method)
at org.apache.hadoop.util.NativeCrc32.calculateChunkedSumsByteArray(NativeCrc32.java:86)
at org.apache.hadoop.util.DataChecksum.calculateChunkedSums(DataChecksum.java:430)
at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:202)
at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:163)
at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1993)
at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1946)
at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
at org.apache.tez.dag.history.recovery.RecoveryService.handle(RecoveryService.java:286)
at org.apache.tez.dag.history.HistoryEventHandler.handleCriticalEvent(HistoryEventHandler.java:102)
at org.apache.tez.dag.app.DAGAppMaster.startDAG(DAGAppMaster.java:2153)
at org.apache.tez.dag.app.DAGAppMaster.startDAG(DAGAppMaster.java:2103)
at org.apache.tez.dag.app.DAGAppMaster.serviceStart(DAGAppMaster.java:1798)
at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193)
at org.apache.tez.dag.app.DAGAppMaster$6.run(DAGAppMaster.java:2224)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
at org.apache.tez.dag.app.DAGAppMaster.initAndStartAppMaster(DAGAppMaster.java:2220)
at org.apache.tez.dag.app.DAGAppMaster.main(DAGAppMaster.java:2038)
________________________________
r7raul1984@163.com<ma...@163.com>
From: r7raul1984@163.com<ma...@163.com>
Date: 2015-06-04 13:19
To: user<ma...@tez.apache.org>
Subject: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
hive> set hive.execution.engine=tez;
hive> set hive.tez.exec.print.summary=true;
hive> select userid,count(*) from u_data group by userid order by userid;
Query ID = hdfs_20150604131717_6863a8c8-c3a6-4865-b098-06aff509dbf8
Total jobs = 1
Launching Job 1 out of 1
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(ATSHistoryACLPolicyManager.java:120)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createSessionDomain(ATSHistoryACLPolicyManager.java:167)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionACLs(ATSHistoryACLPolicyManager.java:228)
at org.apache.tez.client.TezClientUtils.createApplicationSubmissionContext(TezClientUtils.java:525)
at org.apache.tez.client.TezClient.start(TezClient.java:352)
at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:191)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.updateSession(TezTask.java:234)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:136)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160)
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:88)
at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1638)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1397)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1183)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1039)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:207)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:159)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:370)
at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:754)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.main(RunJar.java:212)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.api.records.timeline.TimelineDomain
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 26 more
FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.tez.TezTask. org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
hive>
________________________________
r7raul1984@163.com<ma...@163.com>
Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
Posted by "r7raul1984@163.com" <r7...@163.com>.
I commet
<!-- <property>
<description>Log history using the Timeline Server</description>
<name>tez.history.logging.service.class</name>
<value>org.apache.tez.dag.history.logging.ats.ATSHistoryLoggingService</value>
</property>
<property>
<description>Publish configuration information to Timeline server.-XX:+UseParallelGC -XX:+UseG1GC </description>
<name>tez.runtime.convert.user-payload.to.history-text</name>
<value>true</value>
</property>-->
The problem not occur.
Then I try run hadoop jar tez-examples-0.7.0.jar orderedwordcount /usr/tez/tez-site.xml /tmp . Return error:
Stack trace: ExitCodeException exitCode=1:
at org.apache.hadoop.util.Shell.runCommand(Shell.java:538)
at org.apache.hadoop.util.Shell.run(Shell.java:455)
at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:702)
at org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor.launchContainer(DefaultContainerExecutor.java:196)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:299)
at org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.call(ContainerLaunch.java:81)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
I check log for detail ,I found:
2015-06-04 13:46:04,218 INFO [main] app.DAGAppMaster: Running DAG: OrderedWordCount
2015-06-04 13:46:04,263 ERROR [main] app.DAGAppMaster: Error starting DAGAppMaster
java.lang.UnsatisfiedLinkError: org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(II[BI[BIILjava/lang/String;JZ)V
at org.apache.hadoop.util.NativeCrc32.nativeComputeChunkedSumsByteArray(Native Method)
at org.apache.hadoop.util.NativeCrc32.calculateChunkedSumsByteArray(NativeCrc32.java:86)
at org.apache.hadoop.util.DataChecksum.calculateChunkedSums(DataChecksum.java:430)
at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunks(FSOutputSummer.java:202)
at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:163)
at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1993)
at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:1946)
at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
at org.apache.tez.dag.history.recovery.RecoveryService.handle(RecoveryService.java:286)
at org.apache.tez.dag.history.HistoryEventHandler.handleCriticalEvent(HistoryEventHandler.java:102)
at org.apache.tez.dag.app.DAGAppMaster.startDAG(DAGAppMaster.java:2153)
at org.apache.tez.dag.app.DAGAppMaster.startDAG(DAGAppMaster.java:2103)
at org.apache.tez.dag.app.DAGAppMaster.serviceStart(DAGAppMaster.java:1798)
at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193)
at org.apache.tez.dag.app.DAGAppMaster$6.run(DAGAppMaster.java:2224)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:415)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1628)
at org.apache.tez.dag.app.DAGAppMaster.initAndStartAppMaster(DAGAppMaster.java:2220)
at org.apache.tez.dag.app.DAGAppMaster.main(DAGAppMaster.java:2038)
r7raul1984@163.com
From: r7raul1984@163.com
Date: 2015-06-04 13:19
To: user
Subject: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
hive> set hive.execution.engine=tez;
hive> set hive.tez.exec.print.summary=true;
hive> select userid,count(*) from u_data group by userid order by userid;
Query ID = hdfs_20150604131717_6863a8c8-c3a6-4865-b098-06aff509dbf8
Total jobs = 1
Launching Job 1 out of 1
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(ATSHistoryACLPolicyManager.java:120)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createSessionDomain(ATSHistoryACLPolicyManager.java:167)
at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionACLs(ATSHistoryACLPolicyManager.java:228)
at org.apache.tez.client.TezClientUtils.createApplicationSubmissionContext(TezClientUtils.java:525)
at org.apache.tez.client.TezClient.start(TezClient.java:352)
at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:191)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.updateSession(TezTask.java:234)
at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:136)
at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160)
at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:88)
at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1638)
at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1397)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1183)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1039)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:207)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:159)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:370)
at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:754)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.main(RunJar.java:212)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.api.records.timeline.TimelineDomain
at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
... 26 more
FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.tez.TezTask. org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
hive>
r7raul1984@163.com
Re: hive 1.1.0 tez0.7 hadoop 2.5.0 run query NoClassDefFoundError
Posted by Hitesh Shah <hi...@apache.org>.
If you compiled Tez against hadoop-2.6.0 and are deploying it on a hadoop-2.5.0 cluster, you should disable tez acls as YARN timeline in 2.5.0 does not support ACLs.
Please set tez.am.acls.enabled to false as the Timeline layer is trying to enforce acls for the history data.
thanks
— Hitesh
On Jun 3, 2015, at 10:19 PM, r7raul1984@163.com wrote:
> hive> set hive.execution.engine=tez;
> hive> set hive.tez.exec.print.summary=true;
> hive> select userid,count(*) from u_data group by userid order by userid;
> Query ID = hdfs_20150604131717_6863a8c8-c3a6-4865-b098-06aff509dbf8
> Total jobs = 1
> Launching Job 1 out of 1
> java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
> at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(ATSHistoryACLPolicyManager.java:120)
> at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createSessionDomain(ATSHistoryACLPolicyManager.java:167)
> at org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionACLs(ATSHistoryACLPolicyManager.java:228)
> at org.apache.tez.client.TezClientUtils.createApplicationSubmissionContext(TezClientUtils.java:525)
> at org.apache.tez.client.TezClient.start(TezClient.java:352)
> at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:191)
> at org.apache.hadoop.hive.ql.exec.tez.TezTask.updateSession(TezTask.java:234)
> at org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(TezTask.java:136)
> at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:160)
> at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:88)
> at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:1638)
> at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1397)
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1183)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1049)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1039)
> at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:207)
> at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:159)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:370)
> at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:754)
> at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:675)
> at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:615)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:212)
> Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.yarn.api.records.timeline.TimelineDomain
> at java.net.URLClassLoader$1.run(URLClassLoader.java:366)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:355)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:354)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:425)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:358)
> ... 26 more
> FAILED: Execution Error, return code -101 from org.apache.hadoop.hive.ql.exec.tez.TezTask. org/apache/hadoop/yarn/api/records/timeline/TimelineDomain
> hive>
>
> r7raul1984@163.com