oozie java action issue

[Immanuel Fredrick <im...@xoanonanalytics.com>]

2016-03-09 03:54:32,070 ERROR [eventHandlingThread]
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Error
writing History Event:
org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinishedEvent@7594e5ec
java.nio.channels.ClosedChannelException
at
org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1765)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:108)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.codehaus.jackson.impl.Utf8Generator._flushBuffer(Utf8Generator.java:1754)
at org.codehaus.jackson.impl.Utf8Generator.flush(Utf8Generator.java:1088)
at org.apache.avro.io.JsonEncoder.flush(JsonEncoder.java:73)
at
org.apache.hadoop.mapreduce.jobhistory.EventWriter.write(EventWriter.java:84)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler$MetaInfo.writeEvent(JobHistoryEventHandler.java:1281)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler.handleEvent(JobHistoryEventHandler.java:570)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler$1.run(JobHistoryEventHandler.java:326)
at java.lang.Thread.run(Thread.java:745)
2016-03-09 03:54:32,070 INFO [AsyncDispatcher event handler]
org.apache.hadoop.mapreduce.v2.app.job.impl.TaskImpl: Task succeeded with
attempt attempt_1456313053210_0589_m_000000_0
2016-03-09 03:54:32,071 ERROR [eventHandlingThread]
org.apache.hadoop.yarn.YarnUncaughtExceptionHandler: Thread
Thread[eventHandlingThread,5,main] threw an Exception.
org.apache.hadoop.yarn.exceptions.YarnRuntimeException:
java.nio.channels.ClosedChannelException
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler.handleEvent(JobHistoryEventHandler.java:585)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler$1.run(JobHistoryEventHandler.java:326)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.nio.channels.ClosedChannelException
at
org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1765)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:108)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.codehaus.jackson.impl.Utf8Generator._flushBuffer(Utf8Generator.java:1754)
at org.codehaus.jackson.impl.Utf8Generator.flush(Utf8Generator.java:1088)
at org.apache.avro.io.JsonEncoder.flush(JsonEncoder.java:73)
at
org.apache.hadoop.mapreduce.jobhistory.EventWriter.write(EventWriter.java:84)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler$MetaInfo.writeEvent(JobHistoryEventHandler.java:1281)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler.handleEvent(JobHistoryEventHandler.java:570)
... 2 more
2016-03-09 03:54:32,072 INFO [AsyncDispatcher event handler]
org.apache.hadoop.mapreduce.v2.app.job.impl.TaskImpl:
task_1456313053210_0589_m_000000 Task Transitioned from RUNNING to SUCCEEDED
2016-03-09 03:54:32,074 INFO [AsyncDispatcher event handler]
org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl: Num completed Tasks: 1
2016-03-09 03:54:32,074 INFO [AsyncDispatcher event handler]
org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl:
job_1456313053210_0589Job Transitioned from RUNNING to COMMITTING
2016-03-09 03:54:32,075 INFO [CommitterEvent Processor #1]
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler: Processing
the event EventType: JOB_COMMIT
2016-03-09 03:54:32,075 ERROR [CommitterEvent Processor #1]
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler: could not
create failure file.
java.io.IOException: Filesystem closed
at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:837)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1720)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1662)
at
org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:404)
at
org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:400)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:400)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:343)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:917)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:898)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:795)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.touchz(CommitterEventHandler.java:265)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.handleJobCommit(CommitterEventHandler.java:280)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.run(CommitterEventHandler.java:237)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2016-03-09 03:54:32,075 ERROR [CommitterEvent Processor #1]
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler: Could not
commit job
java.io.IOException: Filesystem closed
at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:837)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1720)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1662)
at
org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:404)
at
org.apache.hadoop.hdfs.DistributedFileSystem$6.doCall(DistributedFileSystem.java:400)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:400)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:343)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:917)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:898)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:795)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.touchz(CommitterEventHandler.java:265)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.handleJobCommit(CommitterEventHandler.java:271)
at
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler$EventProcessor.run(CommitterEventHandler.java:237)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
2016-03-09 03:54:32,077 INFO [AsyncDispatcher event handler]
org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl:
job_1456313053210_0589Job Transitioned from COMMITTING to FAIL_ABORT
2016-03-09 03:54:32,077 INFO [CommitterEvent Processor #2]
org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler: Processing
the event EventType: JOB_ABORT
2016-03-09 03:54:32,077 WARN [uber-SubtaskRunner]
org.apache.hadoop.mapred.LocalContainerLauncher: Unable to delete
unexpected local file/dir .action.xml.crc: insufficient permissions?
2016-03-09 03:54:32,078 INFO [uber-SubtaskRunner]
org.apache.hadoop.mapred.LocalContainerLauncher: removed attempt
attempt_1456313053210_0589_m_000000_0 from the futures to keep track of
2016-03-09 03:54:32,078 INFO [AsyncDispatcher event handler]
org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl:
attempt_1456313053210_0589_m_000000_0 TaskAttempt Transitioned from
SUCCESS_FINISHING_CONTAINER to SUCCEEDED
2016-03-09 03:54:32,078 INFO [uber-EventHandler]
org.apache.hadoop.mapred.LocalContainerLauncher: Processing the event
EventType: CONTAINER_COMPLETED for container
container_1456313053210_0589_01_000001 taskAttempt
attempt_1456313053210_0589_m_000000_0
2016-03-09 03:54:32,087 INFO [AsyncDispatcher event handler]
org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl:
job_1456313053210_0589Job Transitioned from FAIL_ABORT to FAILED
2016-03-09 03:54:32,088 INFO [Thread-71]
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: We are finishing cleanly so
this is the last retry
2016-03-09 03:54:32,088 INFO [Thread-71]
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Notify RMCommunicator
isAMLastRetry: true
2016-03-09 03:54:32,088 INFO [Thread-71]
org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator: RMCommunicator
notified that shouldUnregistered is: true
2016-03-09 03:54:32,088 INFO [Thread-71]
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Notify JHEH isAMLastRetry:
true
2016-03-09 03:54:32,088 INFO [Thread-71]
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler:
JobHistoryEventHandler notified that forceJobCompletion is true
2016-03-09 03:54:32,088 INFO [Thread-71]
org.apache.hadoop.mapreduce.v2.app.MRAppMaster: Calling stop for all the
services
2016-03-09 03:54:32,089 INFO [Thread-71]
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Stopping
JobHistoryEventHandler. Size of the outstanding queue size is 2
2016-03-09 03:54:32,089 INFO [Thread-71]
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: In stop,
writing event TASK_FINISHED
2016-03-09 03:54:32,092 ERROR [Thread-71]
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler: Error
writing History Event:
org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent@cc0b3ef
java.nio.channels.ClosedChannelException
at
org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1765)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:108)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.codehaus.jackson.impl.Utf8Generator._flushBuffer(Utf8Generator.java:1754)
at org.codehaus.jackson.impl.Utf8Generator.flush(Utf8Generator.java:1088)
at org.apache.avro.io.JsonEncoder.flush(JsonEncoder.java:73)
at
org.apache.hadoop.mapreduce.jobhistory.EventWriter.write(EventWriter.java:84)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler$MetaInfo.writeEvent(JobHistoryEventHandler.java:1281)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler.handleEvent(JobHistoryEventHandler.java:570)
at
org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler.serviceStop(JobHistoryEventHandler.java:382)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:221)
at
org.apache.hadoop.service.ServiceOperations.stop(ServiceOperations.java:52)
at
org.apache.hadoop.service.ServiceOperations.stopQuietly(ServiceOperations.java:80)
at
org.apache.hadoop.service.CompositeService.stop(CompositeService.java:157)
at
org.apache.hadoop.service.CompositeService.serviceStop(CompositeService.java:131)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.serviceStop(MRAppMaster.java:1629)
at org.apache.hadoop.service.AbstractService.stop(AbstractService.java:221)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.stop(MRAppMaster.java:1125)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster.shutDownJob(MRAppMaster.java:567)
at
org.apache.hadoop.mapreduce.v2.app.MRAppMaster$JobFinishEventHandler$1.run(MRAppMaster.java:614)

-- 
Regards,
Immanuel