You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@kafka.apache.org by Manikumar Reddy <ku...@nmsworks.co.in> on 2015/03/02 05:24:15 UTC
Broker shuts down due to unrecoverable I/O error
Kafka 0.8.2 server got stopped after getting below I/O exception.
Any thoughts on below exception? Can it be file system related?
[2015-03-01 14:36:27,627] FATAL [KafkaApi-0] Halting due to unrecoverable
I/O error while handling produce request: (kafka.serv
er.KafkaApis)
kafka.common.KafkaStorageException: I/O exception in append to log
'NOTIFICATION_CATEGORY'
at kafka.log.Log.append(Log.scala:266)
at
kafka.cluster.Partition$$anonfun$appendMessagesToLeader$1.apply(Partition.scala:379)
at
kafka.cluster.Partition$$anonfun$appendMessagesToLeader$1.apply(Partition.scala:365)
at kafka.utils.Utils$.inLock(Utils.scala:535)
at kafka.utils.Utils$.inReadLock(Utils.scala:541)
at
kafka.cluster.Partition.appendMessagesToLeader(Partition.scala:365)
at
kafka.server.KafkaApis$$anonfun$appendToLocalLog$2.apply(KafkaApis.scala:291)
at
kafka.server.KafkaApis$$anonfun$appendToLocalLog$2.apply(KafkaApis.scala:282)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
at
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
at
scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
at
scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
at scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
at
scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
at scala.collection.AbstractTraversable.map(Traversable.scala:105)
at kafka.server.KafkaApis.appendToLocalLog(KafkaApis.scala:282)
at
kafka.server.KafkaApis.handleProducerOrOffsetCommitRequest(KafkaApis.scala:204)
at kafka.server.KafkaApis.handle(KafkaApis.scala:59)
at
kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.FileNotFoundException:
/KAFKA/logs_kafka/NOTIFICATION_CATEGORY-0/00000000000003746532.index (No
such file or directory)
at java.io.RandomAccessFile.open(Native Method)
at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241)
at
kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:277)
at
kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:276)
at kafka.utils.Utils$.inLock(Utils.scala:535)
at kafka.log.OffsetIndex.resize(OffsetIndex.scala:276)
at
kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply$mcV$sp(OffsetIndex.scala:265)
at
kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
at
kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
at kafka.utils.Utils$.inLock(Utils.scala:535)
at kafka.log.OffsetIndex.trimToValidSize(OffsetIndex.scala:264)
at kafka.log.Log.roll(Log.scala:563)
at kafka.log.Log.maybeRoll(Log.scala:539)
at kafka.log.Log.append(Log.scala:306)
There are other exceptions also:
[2015-03-01 13:59:18,616] ERROR Uncaught exception in scheduled task
'kafka-log-retention' (kafka.utils.KafkaScheduler)
java.io.FileNotFoundException:
/cygnet/KAFKA/logs_kafka/NOTIF_CH_PROBMGMT_RESOURCE-0/00000000000000458729.index
(No such file or
directory)
at java.io.RandomAccessFile.open(Native Method)
at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241)
at
kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:277)
at
kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:276)
at kafka.utils.Utils$.inLock(Utils.scala:535)
at kafka.log.OffsetIndex.resize(OffsetIndex.scala:276)
at
kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply$mcV$sp(OffsetIndex.scala:265)
at
kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
at
kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
at kafka.utils.Utils$.inLock(Utils.scala:535)
at kafka.log.OffsetIndex.trimToValidSize(OffsetIndex.scala:264)
at kafka.log.Log.roll(Log.scala:563)
at kafka.log.Log.deleteOldSegments(Log.scala:486)
at
kafka.log.LogManager.kafka$log$LogManager$$cleanupExpiredSegments(LogManager.scala:411)
at
kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:442)
at
kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:440)
at
scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at
scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
at
scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
at kafka.log.LogManager.cleanupLogs(LogManager.scala:440)
at
kafka.log.LogManager$$anonfun$startup$1.apply$mcV$sp(LogManager.scala:182)
at
kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:99)
at kafka.utils.Utils$$anon$1.run(Utils.scala:54)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
Re: Broker shuts down due to unrecoverable I/O error
Posted by Manikumar Reddy <ku...@nmsworks.co.in>.
Hi,
We are running on RedHat Linux with SAN storage. This happened only once.
Thanks,
Manikumar.
On Tue, Mar 3, 2015 at 10:02 PM, Jun Rao <ju...@confluent.io> wrote:
> Which OS is this on? Is this easily reproducible?
>
> Thanks,
>
> Jun
>
> On Sun, Mar 1, 2015 at 8:24 PM, Manikumar Reddy <ku...@nmsworks.co.in>
> wrote:
>
> > Kafka 0.8.2 server got stopped after getting below I/O exception.
> > Any thoughts on below exception? Can it be file system related?
> >
> > [2015-03-01 14:36:27,627] FATAL [KafkaApi-0] Halting due to unrecoverable
> > I/O error while handling produce request: (kafka.serv
> > er.KafkaApis)
> > kafka.common.KafkaStorageException: I/O exception in append to log
> > 'NOTIFICATION_CATEGORY'
> > at kafka.log.Log.append(Log.scala:266)
> > at
> >
> >
> kafka.cluster.Partition$$anonfun$appendMessagesToLeader$1.apply(Partition.scala:379)
> > at
> >
> >
> kafka.cluster.Partition$$anonfun$appendMessagesToLeader$1.apply(Partition.scala:365)
> > at kafka.utils.Utils$.inLock(Utils.scala:535)
> > at kafka.utils.Utils$.inReadLock(Utils.scala:541)
> > at
> > kafka.cluster.Partition.appendMessagesToLeader(Partition.scala:365)
> > at
> >
> >
> kafka.server.KafkaApis$$anonfun$appendToLocalLog$2.apply(KafkaApis.scala:291)
> > at
> >
> >
> kafka.server.KafkaApis$$anonfun$appendToLocalLog$2.apply(KafkaApis.scala:282)
> > at
> >
> >
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> > at
> >
> >
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> > at
> >
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
> > at
> >
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
> > at
> >
> scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
> > at
> scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
> > at scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
> > at
> > scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
> > at
> scala.collection.AbstractTraversable.map(Traversable.scala:105)
> > at kafka.server.KafkaApis.appendToLocalLog(KafkaApis.scala:282)
> > at
> >
> >
> kafka.server.KafkaApis.handleProducerOrOffsetCommitRequest(KafkaApis.scala:204)
> > at kafka.server.KafkaApis.handle(KafkaApis.scala:59)
> > at
> > kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59)
> > at java.lang.Thread.run(Thread.java:745)
> > Caused by: java.io.FileNotFoundException:
> > /KAFKA/logs_kafka/NOTIFICATION_CATEGORY-0/00000000000003746532.index (No
> > such file or directory)
> > at java.io.RandomAccessFile.open(Native Method)
> > at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241)
> > at
> > kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:277)
> > at
> > kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:276)
> > at kafka.utils.Utils$.inLock(Utils.scala:535)
> > at kafka.log.OffsetIndex.resize(OffsetIndex.scala:276)
> > at
> >
> >
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply$mcV$sp(OffsetIndex.scala:265)
> > at
> >
> >
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> > at
> >
> >
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> > at kafka.utils.Utils$.inLock(Utils.scala:535)
> > at kafka.log.OffsetIndex.trimToValidSize(OffsetIndex.scala:264)
> > at kafka.log.Log.roll(Log.scala:563)
> > at kafka.log.Log.maybeRoll(Log.scala:539)
> > at kafka.log.Log.append(Log.scala:306)
> >
> >
> > There are other exceptions also:
> >
> > [2015-03-01 13:59:18,616] ERROR Uncaught exception in scheduled task
> > 'kafka-log-retention' (kafka.utils.KafkaScheduler)
> > java.io.FileNotFoundException:
> >
> >
> /cygnet/KAFKA/logs_kafka/NOTIF_CH_PROBMGMT_RESOURCE-0/00000000000000458729.index
> > (No such file or
> > directory)
> > at java.io.RandomAccessFile.open(Native Method)
> > at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241)
> > at
> > kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:277)
> > at
> > kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:276)
> > at kafka.utils.Utils$.inLock(Utils.scala:535)
> > at kafka.log.OffsetIndex.resize(OffsetIndex.scala:276)
> > at
> >
> >
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply$mcV$sp(OffsetIndex.scala:265)
> > at
> >
> >
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> > at
> >
> >
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> > at kafka.utils.Utils$.inLock(Utils.scala:535)
> > at kafka.log.OffsetIndex.trimToValidSize(OffsetIndex.scala:264)
> > at kafka.log.Log.roll(Log.scala:563)
> > at kafka.log.Log.deleteOldSegments(Log.scala:486)
> > at
> >
> >
> kafka.log.LogManager.kafka$log$LogManager$$cleanupExpiredSegments(LogManager.scala:411)
> > at
> > kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:442)
> > at
> > kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:440)
> > at
> >
> >
> scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
> > at scala.collection.Iterator$class.foreach(Iterator.scala:727)
> > at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
> > at
> > scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
> > at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
> > at
> >
> >
> scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
> > at kafka.log.LogManager.cleanupLogs(LogManager.scala:440)
> > at
> >
> kafka.log.LogManager$$anonfun$startup$1.apply$mcV$sp(LogManager.scala:182)
> > at
> >
> kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:99)
> > at kafka.utils.Utils$$anon$1.run(Utils.scala:54)
> > at
> > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> > at
> java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
> > at
> >
> >
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
> > at
> >
> >
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
> > at
> >
> >
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> >
>
Re: Broker shuts down due to unrecoverable I/O error
Posted by Jun Rao <ju...@confluent.io>.
Which OS is this on? Is this easily reproducible?
Thanks,
Jun
On Sun, Mar 1, 2015 at 8:24 PM, Manikumar Reddy <ku...@nmsworks.co.in>
wrote:
> Kafka 0.8.2 server got stopped after getting below I/O exception.
> Any thoughts on below exception? Can it be file system related?
>
> [2015-03-01 14:36:27,627] FATAL [KafkaApi-0] Halting due to unrecoverable
> I/O error while handling produce request: (kafka.serv
> er.KafkaApis)
> kafka.common.KafkaStorageException: I/O exception in append to log
> 'NOTIFICATION_CATEGORY'
> at kafka.log.Log.append(Log.scala:266)
> at
>
> kafka.cluster.Partition$$anonfun$appendMessagesToLeader$1.apply(Partition.scala:379)
> at
>
> kafka.cluster.Partition$$anonfun$appendMessagesToLeader$1.apply(Partition.scala:365)
> at kafka.utils.Utils$.inLock(Utils.scala:535)
> at kafka.utils.Utils$.inReadLock(Utils.scala:541)
> at
> kafka.cluster.Partition.appendMessagesToLeader(Partition.scala:365)
> at
>
> kafka.server.KafkaApis$$anonfun$appendToLocalLog$2.apply(KafkaApis.scala:291)
> at
>
> kafka.server.KafkaApis$$anonfun$appendToLocalLog$2.apply(KafkaApis.scala:282)
> at
>
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at
>
> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
> at
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
> at
> scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:98)
> at
> scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:226)
> at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39)
> at scala.collection.mutable.HashMap.foreach(HashMap.scala:98)
> at
> scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
> at scala.collection.AbstractTraversable.map(Traversable.scala:105)
> at kafka.server.KafkaApis.appendToLocalLog(KafkaApis.scala:282)
> at
>
> kafka.server.KafkaApis.handleProducerOrOffsetCommitRequest(KafkaApis.scala:204)
> at kafka.server.KafkaApis.handle(KafkaApis.scala:59)
> at
> kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.io.FileNotFoundException:
> /KAFKA/logs_kafka/NOTIFICATION_CATEGORY-0/00000000000003746532.index (No
> such file or directory)
> at java.io.RandomAccessFile.open(Native Method)
> at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241)
> at
> kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:277)
> at
> kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:276)
> at kafka.utils.Utils$.inLock(Utils.scala:535)
> at kafka.log.OffsetIndex.resize(OffsetIndex.scala:276)
> at
>
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply$mcV$sp(OffsetIndex.scala:265)
> at
>
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> at
>
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> at kafka.utils.Utils$.inLock(Utils.scala:535)
> at kafka.log.OffsetIndex.trimToValidSize(OffsetIndex.scala:264)
> at kafka.log.Log.roll(Log.scala:563)
> at kafka.log.Log.maybeRoll(Log.scala:539)
> at kafka.log.Log.append(Log.scala:306)
>
>
> There are other exceptions also:
>
> [2015-03-01 13:59:18,616] ERROR Uncaught exception in scheduled task
> 'kafka-log-retention' (kafka.utils.KafkaScheduler)
> java.io.FileNotFoundException:
>
> /cygnet/KAFKA/logs_kafka/NOTIF_CH_PROBMGMT_RESOURCE-0/00000000000000458729.index
> (No such file or
> directory)
> at java.io.RandomAccessFile.open(Native Method)
> at java.io.RandomAccessFile.<init>(RandomAccessFile.java:241)
> at
> kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:277)
> at
> kafka.log.OffsetIndex$$anonfun$resize$1.apply(OffsetIndex.scala:276)
> at kafka.utils.Utils$.inLock(Utils.scala:535)
> at kafka.log.OffsetIndex.resize(OffsetIndex.scala:276)
> at
>
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply$mcV$sp(OffsetIndex.scala:265)
> at
>
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> at
>
> kafka.log.OffsetIndex$$anonfun$trimToValidSize$1.apply(OffsetIndex.scala:265)
> at kafka.utils.Utils$.inLock(Utils.scala:535)
> at kafka.log.OffsetIndex.trimToValidSize(OffsetIndex.scala:264)
> at kafka.log.Log.roll(Log.scala:563)
> at kafka.log.Log.deleteOldSegments(Log.scala:486)
> at
>
> kafka.log.LogManager.kafka$log$LogManager$$cleanupExpiredSegments(LogManager.scala:411)
> at
> kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:442)
> at
> kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:440)
> at
>
> scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772)
> at scala.collection.Iterator$class.foreach(Iterator.scala:727)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
> at
> scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
> at
>
> scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771)
> at kafka.log.LogManager.cleanupLogs(LogManager.scala:440)
> at
> kafka.log.LogManager$$anonfun$startup$1.apply$mcV$sp(LogManager.scala:182)
> at
> kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:99)
> at kafka.utils.Utils$$anon$1.run(Utils.scala:54)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
> at
>
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
> at
>
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
> at
>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>