You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@accumulo.apache.org by Roshan Punnoose <ro...@gmail.com> on 2014/05/28 00:19:55 UTC
HDFS Closed Channel
Has anyone seen a case were a large parallel scan in mapreduce causes HDFS
to close on the tablets? My accumulo instance (version 1.5.1) seems to be
dying during an intensive mapreduce job. These are the only errors I see in
the tablet server logs:
2014-05-27 10:59:34,822 [log.DfsLogger] ERROR:
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1320)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:98)
at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:59)
at java.io.DataOutputStream.writeLong(DataOutputStream.java:224)
at org.apache.accumulo.server.logger.LogFileKey.write(LogFileKey.java:108)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.write(DfsLogger.java:435)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.logFileData(DfsLogger.java:448)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.logManyTablets(DfsLogger.java:479)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.log(DfsLogger.java:440)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger$5.write(TabletServerLogger.java:345)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:273)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
at com.sun.proxy.$Proxy11.update(Unknown Source)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
at java.lang.Thread.run(Thread.java:744)
2014-05-27 10:59:34,826 [log.DfsLogger] WARN : Exception syncing
java.lang.reflect.InvocationTargetException
2014-05-27 10:59:34,827 [log.TabletServerLogger] ERROR: Unexpected
error writing to log, retrying attempt 1
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
at org.apache.accumulo.server.tabletserver.log.DfsLogger$LoggerOperation.await(DfsLogger.java:178)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:279)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
at com.sun.proxy.$Proxy11.update(Unknown Source)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor8.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.server.tabletserver.log.DfsLogger$LogSyncingTask.run(DfsLogger.java:118)
... 1 more
Caused by: java.io.IOException: Filesystem closed
at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:629)
at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1635)
at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1618)
at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1601)
at org.apache.hadoop.fs.FSDataOutputStream.hsync(FSDataOutputStream.java:137)
... 5 more
2014-05-27 10:59:34,881 [log.DfsLogger] ERROR:
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1320)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:98)
at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:59)
at java.io.DataOutputStream.writeLong(DataOutputStream.java:224)
at org.apache.accumulo.server.logger.LogFileKey.write(LogFileKey.java:108)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.write(DfsLogger.java:435)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.logFileData(DfsLogger.java:448)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.logManyTablets(DfsLogger.java:479)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.log(DfsLogger.java:440)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger$5.write(TabletServerLogger.java:345)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:273)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
at com.sun.proxy.$Proxy11.update(Unknown Source)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
at java.lang.Thread.run(Thread.java:744)
2014-05-27 10:59:34,883 [log.DfsLogger] WARN : Exception syncing
java.lang.reflect.InvocationTargetException
2014-05-27 10:59:34,883 [log.TabletServerLogger] ERROR: Unexpected
error writing to log, retrying attempt 1
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
at org.apache.accumulo.server.tabletserver.log.DfsLogger$LoggerOperation.await(DfsLogger.java:178)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:279)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
at com.sun.proxy.$Proxy11.update(Unknown Source)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor8.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.server.tabletserver.log.DfsLogger$LogSyncingTask.run(DfsLogger.java:118)
... 1 more
Caused by: java.io.IOException: Filesystem closed
at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:629)
at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1635)
at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1618)
at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1601)
at org.apache.hadoop.fs.FSDataOutputStream.hsync(FSDataOutputStream.java:137)
... 5 more
2014-05-27 10:59:34,885 [log.DfsLogger] ERROR:
java.nio.channels.ClosedChannelException
java.nio.channels.ClosedChannelException
at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:1320)
at org.apache.hadoop.fs.FSOutputSummer.write(FSOutputSummer.java:98)
at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:59)
at java.io.DataOutputStream.writeLong(DataOutputStream.java:224)
at org.apache.accumulo.server.logger.LogFileKey.write(LogFileKey.java:108)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.write(DfsLogger.java:435)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.logFileData(DfsLogger.java:448)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.logManyTablets(DfsLogger.java:479)
at org.apache.accumulo.server.tabletserver.log.DfsLogger.log(DfsLogger.java:440)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger$5.write(TabletServerLogger.java:345)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:273)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
at com.sun.proxy.$Proxy11.update(Unknown Source)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
at java.lang.Thread.run(Thread.java:744)
2014-05-27 10:59:34,888 [log.DfsLogger] WARN : Exception syncing
java.lang.reflect.InvocationTargetException
2014-05-27 10:59:34,889 [log.TabletServerLogger] ERROR: Unexpected
error writing to log, retrying attempt 1
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
at org.apache.accumulo.server.tabletserver.log.DfsLogger$LoggerOperation.await(DfsLogger.java:178)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:279)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.write(TabletServerLogger.java:229)
at org.apache.accumulo.server.tabletserver.log.TabletServerLogger.log(TabletServerLogger.java:342)
at org.apache.accumulo.server.tabletserver.TabletServer$ThriftClientHandler.update(TabletServer.java:1680)
at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.trace.instrument.thrift.TraceWrap$1.invoke(TraceWrap.java:63)
at com.sun.proxy.$Proxy11.update(Unknown Source)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2125)
at org.apache.accumulo.core.tabletserver.thrift.TabletClientService$Processor$update.getResult(TabletClientService.java:2109)
at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39)
at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39)
at org.apache.accumulo.server.util.TServerUtils$TimedProcessor.process(TServerUtils.java:155)
at org.apache.thrift.server.AbstractNonblockingServer$FrameBuffer.invoke(AbstractNonblockingServer.java:478)
at org.apache.accumulo.server.util.TServerUtils$THsHaServer$Invocation.run(TServerUtils.java:207)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.accumulo.trace.instrument.TraceRunnable.run(TraceRunnable.java:47)
at org.apache.accumulo.core.util.LoggingRunnable.run(LoggingRunnable.java:34)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor8.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.accumulo.server.tabletserver.log.DfsLogger$LogSyncingTask.run(DfsLogger.java:118)
... 1 more
Caused by: java.io.IOException: Filesystem closed
at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:629)
at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:1635)
at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1618)
at org.apache.hadoop.hdfs.DFSOutputStream.hsync(DFSOutputStream.java:1601)
at org.apache.hadoop.fs.FSDataOutputStream.hsync(FSDataOutputStream.java:137)
... 5 more