You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@storm.apache.org by "Robert Joseph Evans (JIRA)" <ji...@apache.org> on 2015/10/13 18:24:05 UTC
[jira] [Commented] (STORM-1109) Worker exists frequently due to
java.net.SocketTimeoutException
[ https://issues.apache.org/jira/browse/STORM-1109?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14955224#comment-14955224 ]
Robert Joseph Evans commented on STORM-1109:
--------------------------------------------
[~kraks],
I think this is the worker and not the supervisor that is crashing.
> Worker exists frequently due to java.net.SocketTimeoutException
> ---------------------------------------------------------------
>
> Key: STORM-1109
> URL: https://issues.apache.org/jira/browse/STORM-1109
> Project: Apache Storm
> Issue Type: Bug
> Components: storm-kafka
> Affects Versions: 0.9.5
> Environment: Prod
> Reporter: rakesh
> Priority: Critical
>
> One of the supervisor is exiting frequently , following is the log
> 015-10-13T20:57:20.245+0530 k.c.SimpleConsumer [INFO] Reconnect due to socket error: null
> 2015-10-13T20:57:30.284+0530 b.s.util [ERROR] Async loop died!
> java.lang.RuntimeException: java.net.SocketTimeoutException
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:146) ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.fill(PartitionManager.java:134) ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.next(PartitionManager.java:108) ~[stormjar.jar:na]
> at storm.kafka.KafkaSpout.nextTuple(KafkaSpout.java:123) ~[stormjar.jar:na]
> at backtype.storm.daemon.executor$fn__6579$fn__6594$fn__6623.invoke(executor.clj:565) ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$async_loop$fn__459.invoke(util.clj:463) ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> Caused by: java.net.SocketTimeoutException: null
> at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:229) ~[na:1.7.0_67]
> at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) ~[na:1.7.0_67]
> at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) ~[na:1.7.0_67]
> at kafka.utils.Utils$.read(Utils.scala:395) ~[stormjar.jar:na]
> at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) ~[stormjar.jar:na]
> at kafka.network.Receive$class.readCompletely(Transmission.scala:56) ~[stormjar.jar:na]
> at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) ~[stormjar.jar:na]
> at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:81) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:110) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110) ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:109) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109) ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:108) ~[stormjar.jar:na]
> at kafka.javaapi.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:48) ~[stormjar.jar:na]
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:141) ~[stormjar.jar:na]
> ... 7 common frames omitted
> 2015-10-13T20:57:30.285+0530 b.s.d.executor [ERROR]
> java.lang.RuntimeException: java.net.SocketTimeoutException
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:146) ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.fill(PartitionManager.java:134) ~[stormjar.jar:na]
> at storm.kafka.PartitionManager.next(PartitionManager.java:108) ~[stormjar.jar:na]
> at storm.kafka.KafkaSpout.nextTuple(KafkaSpout.java:123) ~[stormjar.jar:na]
> at backtype.storm.daemon.executor$fn__6579$fn__6594$fn__6623.invoke(executor.clj:565) ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$async_loop$fn__459.invoke(util.clj:463) ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> Caused by: java.net.SocketTimeoutException: null
> at sun.nio.ch.SocketAdaptor$SocketInputStream.read(SocketAdaptor.java:229) ~[na:1.7.0_67]
> at sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103) ~[na:1.7.0_67]
> at java.nio.channels.Channels$ReadableByteChannelImpl.read(Channels.java:385) ~[na:1.7.0_67]
> at kafka.utils.Utils$.read(Utils.scala:395) ~[stormjar.jar:na]
> at kafka.network.BoundedByteBufferReceive.readFrom(BoundedByteBufferReceive.scala:54) ~[stormjar.jar:na]
> at kafka.network.Receive$class.readCompletely(Transmission.scala:56) ~[stormjar.jar:na]
> at kafka.network.BoundedByteBufferReceive.readCompletely(BoundedByteBufferReceive.scala:29) ~[stormjar.jar:na]
> at kafka.network.BlockingChannel.receive(BlockingChannel.scala:100) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:81) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:71) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SimpleConsumer.scala:110) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1$$anonfun$apply$mcV$sp$1.apply(SimpleConsumer.scala:110) ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply$mcV$sp(SimpleConsumer.scala:109) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer$$anonfun$fetch$1.apply(SimpleConsumer.scala:109) ~[stormjar.jar:na]
> at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:33) ~[stormjar.jar:na]
> at kafka.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:108) ~[stormjar.jar:na]
> at kafka.javaapi.consumer.SimpleConsumer.fetch(SimpleConsumer.scala:48) ~[stormjar.jar:na]
> at storm.kafka.KafkaUtils.fetchMessages(KafkaUtils.java:141) ~[stormjar.jar:na]
> ... 7 common frames omitted
> 2015-10-13T20:57:47.906+0530 b.s.util [ERROR] Halting process: ("Worker died")
> java.lang.RuntimeException: ("Worker died")
> at backtype.storm.util$exit_process_BANG_.doInvoke(util.clj:325) [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.RestFn.invoke(RestFn.java:423) [clojure-1.5.1.jar:na]
> at backtype.storm.daemon.worker$fn__7028$fn__7029.invoke(worker.clj:497) [storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.daemon.executor$mk_executor_data$fn__6480$fn__6481.invoke(executor.clj:240) [storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$async_loop$fn__459.invoke(util.clj:473) [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> 2015-10-13T20:57:47.907+0530 b.s.d.worker [INFO] Shutting down worker
> this is usually accompanied by the following error is supervisor.
> 2015-10-13T20:58:21.858+0530 b.s.d.supervisor [INFO] Shutting down fa862fbe-bfd2-4b53-9abb-cd951303ecb8:42ccaf83-6a17-4dcb-92e6-47416486806d
> 2015-10-13T20:58:21.869+0530 b.s.event [ERROR] Error when processing event
> java.io.IOException: . doesn't exist.
> at org.apache.commons.exec.DefaultExecutor.execute(DefaultExecutor.java:157) ~[commons-exec-1.1.jar:1.1]
> at org.apache.commons.exec.DefaultExecutor.execute(DefaultExecutor.java:147) ~[commons-exec-1.1.jar:1.1]
> at backtype.storm.util$exec_command_BANG_.invoke(util.clj:386) ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$send_signal_to_process.invoke(util.clj:415) ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.util$kill_process_with_sig_term.invoke(util.clj:426) ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.daemon.supervisor$shutdown_worker.invoke(supervisor.clj:197) ~[storm-core-0.9.5.jar:0.9.5]
> at backtype.storm.daemon.supervisor$sync_processes.invoke(supervisor.clj:267) ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.applyToHelper(AFn.java:161) [clojure-1.5.1.jar:na]
> at clojure.lang.AFn.applyTo(AFn.java:151) [clojure-1.5.1.jar:na]
> at clojure.core$apply.invoke(core.clj:619) ~[clojure-1.5.1.jar:na]
> at clojure.core$partial$fn__4190.doInvoke(core.clj:2396) ~[clojure-1.5.1.jar:na]
> at clojure.lang.RestFn.invoke(RestFn.java:397) ~[clojure-1.5.1.jar:na]
> at backtype.storm.event$event_manager$fn__2625.invoke(event.clj:40) ~[storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
> 2015-10-13T20:58:21.877+0530 b.s.util [ERROR] Halting process: ("Error when processing an event")
> java.lang.RuntimeException: ("Error when processing an event")
> at backtype.storm.util$exit_process_BANG_.doInvoke(util.clj:325) [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.RestFn.invoke(RestFn.java:423) [clojure-1.5.1.jar:na]
> at backtype.storm.event$event_manager$fn__2625.invoke(event.clj:48) [storm-core-0.9.5.jar:0.9.5]
> at clojure.lang.AFn.run(AFn.java:24) [clojure-1.5.1.jar:na]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_67]
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)