You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@storm.apache.org by Joaquin Menchaca <jm...@gobalto.com> on 2016/12/12 23:39:41 UTC
NimbusLeaderNotFoundException problem?
*Ideas?*
org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find
leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com"
"nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of
nimbus hosts for config nimbus.seeds?
at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
at clojure.core$some.invoke(core.clj:2570)
at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
at clojure.lang.RestFn.applyTo(RestFn.java:139)
at clojure.core$apply.invoke(core.clj:632)
at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown
Source)
at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown
Source)
at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:745)
--
是故勝兵先勝而後求戰，敗兵先戰而後求勝。
Re: NimbusLeaderNotFoundException problem?
Posted by Joaquin Menchaca <jm...@gobalto.com>.
Alright, found a comment in a post, and put these in my zoo.cfg, they did
the trick:
autopurge.snapRetainCount = 5
autopurge.purgeInterval = 24
On Mon, Dec 12, 2016 at 4:41 PM, Joaquin Menchaca <jm...@gobalto.com>
wrote:
> Diskspace. Newbiew to zk. There's 59 GB of these snapshots in
> /var/lib/zookeeper/version-2
>
> On Mon, Dec 12, 2016 at 4:35 PM, Mostafa Gomaa <mg...@trendak.com> wrote:
>
>> Can you us your zookeeper logs and config?
>>
>> On Dec 13, 2016 2:32 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>> wrote:
>>
>>> I am able to connect to the zookeeper:
>>>
>>> # printf "%s\n" $(echo ruok | nc zookeeper-stormcluster.gobalto.com
>>> 2181)
>>> imok
>>>
>>> But not always. I looked at the zookeeper server, t3.large now, and
>>> the CPU jumps around, sometimes 8%, sometimes 20%-35%, ocassionally 110%.
>>>
>>> I am not sure at this point where to look, it was working up until
>>> recently.
>>>
>>>
>>> On Mon, Dec 12, 2016 at 3:53 PM, Mostafa Gomaa <mg...@trendak.com>
>>> wrote:
>>>
>>>> Yes, your nimbus machine can't connect to zookeeper as per the log.
>>>> Check out that zookeeper is running and that there aren't any networking
>>>> issues between the two machines.
>>>>
>>>> On Dec 13, 2016 1:49 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>>>> wrote:
>>>>
>>>>> Storm.yaml (Storm 1.0.2):
>>>>>
>>>>> storm.zookeeper.servers:
>>>>> - "zookeeper-stormcluster.gobalto.com"
>>>>>
>>>>> nimbus.seeds: ["nimbus-stormcluster.gobalto.com", "
>>>>> nimbus2-stormcluster.gobalto.com"]
>>>>> nimbus.childopts: "-Xmx1024m -Djava.net.preferIPv4Stack=true"
>>>>> ui.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>>>>
>>>>> supervisor.childopts: "-Djava.net.preferIPv4Stack=true"
>>>>> worker.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>>>> supervisor.slots.ports:
>>>>> - 6700
>>>>> - 6701
>>>>> - 6702
>>>>> - 6703
>>>>>
>>>>> storm.local.dir: "/app/storm"
>>>>>
>>>>>
>>>>> Maybe my zookeeper is down or corrupt:
>>>>>
>>>>> 2016-12-12 23:46:50.628 o.a.s.s.o.a.c.ConnectionState [ERROR]
>>>>> Connection timed out for connection string (
>>>>> zookeeper-stormcluster.gobalto.com:2181) and timeout (15000) /
>>>>> elapsed (16327)
>>>>> org.apache.storm.shade.org.apache.curator.CuratorConnectionLossException:
>>>>> KeeperErrorCode = ConnectionLoss
>>>>> at org.apache.storm.shade.org.apache.curator.ConnectionState.ch
>>>>> eckTimeouts(ConnectionState.java:197)
>>>>> at org.apache.storm.shade.org.apache.curator.ConnectionState.ge
>>>>> tZooKeeper(ConnectionState.java:88)
>>>>> at org.apache.storm.shade.org.apache.curator.CuratorZookeeperCl
>>>>> ient.getZooKeeper(CuratorZookeeperClient.java:116)
>>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Cur
>>>>> atorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:489)
>>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:226)
>>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:215)
>>>>> at org.apache.storm.shade.org.apache.curator.RetryLoop.callWith
>>>>> Retry(RetryLoop.java:108)
>>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>>> stsBuilderImpl.pathInForegroundStandard(ExistsBuilderImpl.java:212)
>>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>>> stsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:205)
>>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:168)
>>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:39)
>>>>> at org.apache.storm.zookeeper$exists_node_QMARK_$fn__2090.invok
>>>>> e(zookeeper.clj:109)
>>>>> at org.apache.storm.zookeeper$exists_node_QMARK_.invoke(zookeep
>>>>> er.clj:106)
>>>>> at org.apache.storm.zookeeper$mkdirs.invoke(zookeeper.clj:126)
>>>>> at org.apache.storm.cluster_state.zookeeper_state_factory$_mkSt
>>>>> ate.invoke(zookeeper_state_factory.clj:29)
>>>>> at org.apache.storm.cluster_state.zookeeper_state_factory.mkState(Unknown
>>>>> Source)
>>>>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAcce
>>>>> ssorImpl.java:62)
>>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMe
>>>>> thodAccessorImpl.java:43)
>>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>>> at org.apache.storm.cluster$mk_distributed_cluster_state.doInvo
>>>>> ke(cluster.clj:46)
>>>>> at clojure.lang.RestFn.invoke(RestFn.java:559)
>>>>> at org.apache.storm.cluster$mk_storm_cluster_state.doInvoke(clu
>>>>> ster.clj:250)
>>>>> at clojure.lang.RestFn.invoke(RestFn.java:486)
>>>>> at org.apache.storm.daemon.nimbus$nimbus_data.invoke(nimbus.clj
>>>>> :176)
>>>>> at org.apache.storm.daemon.nimbus$fn__7166$exec_fn__2466__auto_
>>>>> ___7167.invoke(nimbus.clj:1349)
>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:156)
>>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>>> at clojure.core$apply.invoke(core.clj:630)
>>>>> at org.apache.storm.daemon.nimbus$fn__7166$service_handler__741
>>>>> 0.doInvoke(nimbus.clj:1346)
>>>>> at clojure.lang.RestFn.invoke(RestFn.java:421)
>>>>> at org.apache.storm.daemon.nimbus$launch_server_BANG_.invoke(ni
>>>>> mbus.clj:2193)
>>>>> at org.apache.storm.daemon.nimbus$_launch.invoke(nimbus.clj:2226)
>>>>> at org.apache.storm.daemon.nimbus$_main.invoke(nimbus.clj:2249)
>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>>> at org.apache.storm.daemon.nimbus.main(Unknown Source)
>>>>> 2016-12-12 23:46:50.862 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:50.885 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>>> connection established to zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181, initiating session
>>>>> 2016-12-12 23:46:50.886 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>>>> additional data from server sessionid 0x0, likely server has closed socket,
>>>>> closing socket connection and attempting reconnect
>>>>> 2016-12-12 23:46:51.578 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:51.602 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>>> connection established to zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181, initiating session
>>>>> 2016-12-12 23:46:51.603 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>>>> additional data from server sessionid 0x0, likely server has closed socket,
>>>>> closing socket connection and attempting reconnect
>>>>> 2016-12-12 23:46:52.041 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:52.060 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>>> connection established to zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181, initiating session
>>>>> 2016-12-12 23:46:52.061 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>>>> additional data from server sessionid 0x0, likely server has closed socket,
>>>>> closing socket connection and attempting reconnect
>>>>> 2016-12-12 23:46:52.958 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:52.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>>> connection established to zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181, initiating session
>>>>> 2016-12-12 23:46:53.003 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>>>>> unexpected error, closing socket connection and attempting reconnect
>>>>> java.io.IOException: Connection reset by peer
>>>>> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
>>>>> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
>>>>> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
>>>>> at sun.nio.ch.IOUtil.read(IOUtil.java:192)
>>>>> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doIO(ClientCnxnSocketNIO.java:68)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:366)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>> 2016-12-12 23:46:53.130 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:53.146 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server null, unexpected error, closing socket connection and attempting
>>>>> reconnect
>>>>> java.net.ConnectException: Connection refused
>>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>>> .java:717)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>> 2016-12-12 23:46:53.463 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:53.474 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server null, unexpected error, closing socket connection and attempting
>>>>> reconnect
>>>>> java.net.ConnectException: Connection refused
>>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>>> .java:717)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>> 2016-12-12 23:46:54.291 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:54.292 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server null, unexpected error, closing socket connection and attempting
>>>>> reconnect
>>>>> java.net.ConnectException: Connection refused
>>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>>> .java:717)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>> 2016-12-12 23:46:54.440 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:54.454 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server null, unexpected error, closing socket connection and attempting
>>>>> reconnect
>>>>> java.net.ConnectException: Connection refused
>>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>>> .java:717)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>> 2016-12-12 23:46:55.393 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:55.394 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server null, unexpected error, closing socket connection and attempting
>>>>> reconnect
>>>>> java.net.ConnectException: Connection refused
>>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>>> .java:717)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>> 2016-12-12 23:46:55.741 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:55.763 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server null, unexpected error, closing socket connection and attempting
>>>>> reconnect
>>>>> java.net.ConnectException: Connection refused
>>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>>> .java:717)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>> 2016-12-12 23:46:55.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>>> connection to server zookeeper-stormcluster.gobalto
>>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>>> (unknown error)
>>>>> 2016-12-12 23:46:55.969 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0
>>>>> for server null, unexpected error, closing socket connection and attempting
>>>>> reconnect
>>>>> java.net.ConnectException: Connection refused
>>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>>> .java:717)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>>> hread.run(ClientCnxn.java:1081)
>>>>>
>>>>>
>>>>>
>>>>>
>>>>> On Mon, Dec 12, 2016 at 3:45 PM, Mostafa Gomaa <mg...@trendak.com>
>>>>> wrote:
>>>>>
>>>>>> Can you show us your config files as well as the nimbus log?
>>>>>>
>>>>>> On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>>>>>> wrote:
>>>>>>
>>>>>>> Also:
>>>>>>>
>>>>>>> $ storm list
>>>>>>> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
>>>>>>> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
>>>>>>> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
>>>>>>> -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
>>>>>>> -Dstorm.conf.file= -cp /usr/lib/apache/storm/1.0.2/li
>>>>>>> b/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-1
>>>>>>> .3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.jar:/
>>>>>>> usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/lib
>>>>>>> /apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.jar:/usr/lib
>>>>>>> /apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/usr/lib/apache/
>>>>>>> storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr/lib/apache/sto
>>>>>>> rm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache/storm/1.0.2/lib/
>>>>>>> objenesis-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-c
>>>>>>> ore-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/servlet-api-2.5.
>>>>>>> jar:/usr/lib/apache/storm/1.0.2/lib/log4j-over-slf4j-1.6.6.
>>>>>>> jar:/usr/lib/apache/storm/1.0.2/lib/disruptor-3.3.2.jar:/usr
>>>>>>> /lib/apache/storm/1.0.2/lib/reflectasm-1.10.1.jar:/usr/
>>>>>>> lib/apache/storm/1.0.2/lib/asm-5.0.3.jar:/usr/lib/apache/
>>>>>>> storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
>>>>>>> org.apache.storm.command.list
>>>>>>> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>>>>> trying to get leader nimbus info from nimbus-stormcluster.gobalto.co
>>>>>>> m. will retry with a different seed host.
>>>>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>>>>> java.net.ConnectException: Connection refused
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>>>> [clojure-1.7.0.jar:?]
>>>>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>>>>> java.net.ConnectException: Connection refused
>>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> ... 8 more
>>>>>>> Caused by: java.net.ConnectException: Connection refused
>>>>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> ... 8 more
>>>>>>> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>>>>> trying to get leader nimbus info from nimbus2-stormcluster.gobalto.c
>>>>>>> om. will retry with a different seed host.
>>>>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>>>>> java.net.ConnectException: Connection refused
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>>>> [clojure-1.7.0.jar:?]
>>>>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>>>>> java.net.ConnectException: Connection refused
>>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> ... 8 more
>>>>>>> Caused by: java.net.ConnectException: Connection refused
>>>>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>>>>> ~[?:1.8.0_91]
>>>>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>>> ... 8 more
>>>>>>> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
>>>>>>> Could not find leader nimbus from seed hosts ["
>>>>>>> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"].
>>>>>>> Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(Ni
>>>>>>> mbusClient.java:90)
>>>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>>>>
>>>>>>>
>>>>>>> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <
>>>>>>> jmenchaca@gobalto.com> wrote:
>>>>>>>
>>>>>>>>
>>>>>>>> *Ideas?*
>>>>>>>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>>>>>>>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>>>>>>>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>>>>>>>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>>>>>>>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>>>>>>>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>>>>>>>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>>>>>>>> at clojure.core$some.invoke(core.clj:2570)
>>>>>>>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>>>>>>>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>>>>>>>> at clojure.core$apply.invoke(core.clj:632)
>>>>>>>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>>>>>>>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>>>>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>>>>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>>>>>>>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>>>>>>>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>>>>>>>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>>>>>>>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>>>>>>>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>>>>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>>>>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>>>>>>>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>>>>>>>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>>>>>>>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>>>>>>>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>>>>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>>>>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>>>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>>>>>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>>>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>>>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>>>>>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>>>>>>>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>>>>>>> at java.lang.Thread.run(Thread.java:745)
>>>>>>>>
>>>>>>>>
>>>>>>>>
>>>>>>>> --
>>>>>>>>
>>>>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> --
>>>>>>>
>>>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>>>
>>>>>>
>>>>>
>>>>>
>>>>> --
>>>>>
>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>
>>>>
>>>
>>>
>>> --
>>>
>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>
>>
>
>
> --
>
> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>
--
是故勝兵先勝而後求戰，敗兵先戰而後求勝。
Re: NimbusLeaderNotFoundException problem?
Posted by Joaquin Menchaca <jm...@gobalto.com>.
Diskspace. Newbiew to zk. There's 59 GB of these snapshots in
/var/lib/zookeeper/version-2
On Mon, Dec 12, 2016 at 4:35 PM, Mostafa Gomaa <mg...@trendak.com> wrote:
> Can you us your zookeeper logs and config?
>
> On Dec 13, 2016 2:32 AM, "Joaquin Menchaca" <jm...@gobalto.com> wrote:
>
>> I am able to connect to the zookeeper:
>>
>> # printf "%s\n" $(echo ruok | nc zookeeper-stormcluster.gobalto.com 2181)
>> imok
>>
>> But not always. I looked at the zookeeper server, t3.large now, and the
>> CPU jumps around, sometimes 8%, sometimes 20%-35%, ocassionally 110%.
>>
>> I am not sure at this point where to look, it was working up until
>> recently.
>>
>>
>> On Mon, Dec 12, 2016 at 3:53 PM, Mostafa Gomaa <mg...@trendak.com>
>> wrote:
>>
>>> Yes, your nimbus machine can't connect to zookeeper as per the log.
>>> Check out that zookeeper is running and that there aren't any networking
>>> issues between the two machines.
>>>
>>> On Dec 13, 2016 1:49 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>>> wrote:
>>>
>>>> Storm.yaml (Storm 1.0.2):
>>>>
>>>> storm.zookeeper.servers:
>>>> - "zookeeper-stormcluster.gobalto.com"
>>>>
>>>> nimbus.seeds: ["nimbus-stormcluster.gobalto.com", "
>>>> nimbus2-stormcluster.gobalto.com"]
>>>> nimbus.childopts: "-Xmx1024m -Djava.net.preferIPv4Stack=true"
>>>> ui.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>>>
>>>> supervisor.childopts: "-Djava.net.preferIPv4Stack=true"
>>>> worker.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>>> supervisor.slots.ports:
>>>> - 6700
>>>> - 6701
>>>> - 6702
>>>> - 6703
>>>>
>>>> storm.local.dir: "/app/storm"
>>>>
>>>>
>>>> Maybe my zookeeper is down or corrupt:
>>>>
>>>> 2016-12-12 23:46:50.628 o.a.s.s.o.a.c.ConnectionState [ERROR]
>>>> Connection timed out for connection string (
>>>> zookeeper-stormcluster.gobalto.com:2181) and timeout (15000) / elapsed
>>>> (16327)
>>>> org.apache.storm.shade.org.apache.curator.CuratorConnectionLossException:
>>>> KeeperErrorCode = ConnectionLoss
>>>> at org.apache.storm.shade.org.apache.curator.ConnectionState.ch
>>>> eckTimeouts(ConnectionState.java:197)
>>>> at org.apache.storm.shade.org.apache.curator.ConnectionState.ge
>>>> tZooKeeper(ConnectionState.java:88)
>>>> at org.apache.storm.shade.org.apache.curator.CuratorZookeeperCl
>>>> ient.getZooKeeper(CuratorZookeeperClient.java:116)
>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Cur
>>>> atorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:489)
>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:226)
>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:215)
>>>> at org.apache.storm.shade.org.apache.curator.RetryLoop.callWith
>>>> Retry(RetryLoop.java:108)
>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>> stsBuilderImpl.pathInForegroundStandard(ExistsBuilderImpl.java:212)
>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>> stsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:205)
>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:168)
>>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:39)
>>>> at org.apache.storm.zookeeper$exists_node_QMARK_$fn__2090.invok
>>>> e(zookeeper.clj:109)
>>>> at org.apache.storm.zookeeper$exists_node_QMARK_.invoke(zookeep
>>>> er.clj:106)
>>>> at org.apache.storm.zookeeper$mkdirs.invoke(zookeeper.clj:126)
>>>> at org.apache.storm.cluster_state.zookeeper_state_factory$_mkSt
>>>> ate.invoke(zookeeper_state_factory.clj:29)
>>>> at org.apache.storm.cluster_state.zookeeper_state_factory.mkState(Unknown
>>>> Source)
>>>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAcce
>>>> ssorImpl.java:62)
>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMe
>>>> thodAccessorImpl.java:43)
>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>> at org.apache.storm.cluster$mk_distributed_cluster_state.doInvo
>>>> ke(cluster.clj:46)
>>>> at clojure.lang.RestFn.invoke(RestFn.java:559)
>>>> at org.apache.storm.cluster$mk_storm_cluster_state.doInvoke(clu
>>>> ster.clj:250)
>>>> at clojure.lang.RestFn.invoke(RestFn.java:486)
>>>> at org.apache.storm.daemon.nimbus$nimbus_data.invoke(nimbus.clj
>>>> :176)
>>>> at org.apache.storm.daemon.nimbus$fn__7166$exec_fn__2466__auto_
>>>> ___7167.invoke(nimbus.clj:1349)
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:156)
>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>> at clojure.core$apply.invoke(core.clj:630)
>>>> at org.apache.storm.daemon.nimbus$fn__7166$service_handler__741
>>>> 0.doInvoke(nimbus.clj:1346)
>>>> at clojure.lang.RestFn.invoke(RestFn.java:421)
>>>> at org.apache.storm.daemon.nimbus$launch_server_BANG_.invoke(ni
>>>> mbus.clj:2193)
>>>> at org.apache.storm.daemon.nimbus$_launch.invoke(nimbus.clj:2226)
>>>> at org.apache.storm.daemon.nimbus$_main.invoke(nimbus.clj:2249)
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>> at org.apache.storm.daemon.nimbus.main(Unknown Source)
>>>> 2016-12-12 23:46:50.862 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:50.885 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>> connection established to zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181, initiating session
>>>> 2016-12-12 23:46:50.886 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>>> additional data from server sessionid 0x0, likely server has closed socket,
>>>> closing socket connection and attempting reconnect
>>>> 2016-12-12 23:46:51.578 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:51.602 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>> connection established to zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181, initiating session
>>>> 2016-12-12 23:46:51.603 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>>> additional data from server sessionid 0x0, likely server has closed socket,
>>>> closing socket connection and attempting reconnect
>>>> 2016-12-12 23:46:52.041 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:52.060 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>> connection established to zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181, initiating session
>>>> 2016-12-12 23:46:52.061 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>>> additional data from server sessionid 0x0, likely server has closed socket,
>>>> closing socket connection and attempting reconnect
>>>> 2016-12-12 23:46:52.958 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:52.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>>> connection established to zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181, initiating session
>>>> 2016-12-12 23:46:53.003 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181, unexpected
>>>> error, closing socket connection and attempting reconnect
>>>> java.io.IOException: Connection reset by peer
>>>> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
>>>> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
>>>> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
>>>> at sun.nio.ch.IOUtil.read(IOUtil.java:192)
>>>> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doIO(ClientCnxnSocketNIO.java:68)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:366)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>> 2016-12-12 23:46:53.130 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:53.146 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server null, unexpected error, closing socket connection and attempting
>>>> reconnect
>>>> java.net.ConnectException: Connection refused
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>> .java:717)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>> 2016-12-12 23:46:53.463 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:53.474 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server null, unexpected error, closing socket connection and attempting
>>>> reconnect
>>>> java.net.ConnectException: Connection refused
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>> .java:717)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>> 2016-12-12 23:46:54.291 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:54.292 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server null, unexpected error, closing socket connection and attempting
>>>> reconnect
>>>> java.net.ConnectException: Connection refused
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>> .java:717)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>> 2016-12-12 23:46:54.440 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:54.454 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server null, unexpected error, closing socket connection and attempting
>>>> reconnect
>>>> java.net.ConnectException: Connection refused
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>> .java:717)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>> 2016-12-12 23:46:55.393 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:55.394 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server null, unexpected error, closing socket connection and attempting
>>>> reconnect
>>>> java.net.ConnectException: Connection refused
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>> .java:717)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>> 2016-12-12 23:46:55.741 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:55.763 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server null, unexpected error, closing socket connection and attempting
>>>> reconnect
>>>> java.net.ConnectException: Connection refused
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>> .java:717)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>> 2016-12-12 23:46:55.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>>> connection to server zookeeper-stormcluster.gobalto
>>>> .com/10.110.20.8:2181. Will not attempt to authenticate using SASL
>>>> (unknown error)
>>>> 2016-12-12 23:46:55.969 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>>> server null, unexpected error, closing socket connection and attempting
>>>> reconnect
>>>> java.net.ConnectException: Connection refused
>>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>>> .java:717)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>>> hread.run(ClientCnxn.java:1081)
>>>>
>>>>
>>>>
>>>>
>>>> On Mon, Dec 12, 2016 at 3:45 PM, Mostafa Gomaa <mg...@trendak.com>
>>>> wrote:
>>>>
>>>>> Can you show us your config files as well as the nimbus log?
>>>>>
>>>>> On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>>>>> wrote:
>>>>>
>>>>>> Also:
>>>>>>
>>>>>> $ storm list
>>>>>> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
>>>>>> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
>>>>>> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
>>>>>> -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
>>>>>> -Dstorm.conf.file= -cp /usr/lib/apache/storm/1.0.2/li
>>>>>> b/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-1
>>>>>> .3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.jar:/
>>>>>> usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/lib
>>>>>> /apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.jar:/usr/lib
>>>>>> /apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/usr/lib/apache/
>>>>>> storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr/lib/apache/sto
>>>>>> rm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache/storm/1.0.2/
>>>>>> lib/objenesis-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-
>>>>>> core-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/servlet-api-2.
>>>>>> 5.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-over-slf4j-1.6.
>>>>>> 6.jar:/usr/lib/apache/storm/1.0.2/lib/disruptor-3.3.2.jar:/
>>>>>> usr/lib/apache/storm/1.0.2/lib/reflectasm-1.10.1.jar:/
>>>>>> usr/lib/apache/storm/1.0.2/lib/asm-5.0.3.jar:/usr/lib/apache
>>>>>> /storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
>>>>>> org.apache.storm.command.list
>>>>>> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>>>> trying to get leader nimbus info from nimbus-stormcluster.gobalto.com.
>>>>>> will retry with a different seed host.
>>>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>>>> java.net.ConnectException: Connection refused
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>>> [clojure-1.7.0.jar:?]
>>>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>>>> java.net.ConnectException: Connection refused
>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> ... 8 more
>>>>>> Caused by: java.net.ConnectException: Connection refused
>>>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> ... 8 more
>>>>>> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>>>> trying to get leader nimbus info from nimbus2-stormcluster.gobalto.c
>>>>>> om. will retry with a different seed host.
>>>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>>>> java.net.ConnectException: Connection refused
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>>> [clojure-1.7.0.jar:?]
>>>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>>>> java.net.ConnectException: Connection refused
>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> ... 8 more
>>>>>> Caused by: java.net.ConnectException: Connection refused
>>>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>>>> ~[?:1.8.0_91]
>>>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>>> ... 8 more
>>>>>> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
>>>>>> Could not find leader nimbus from seed hosts ["
>>>>>> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"].
>>>>>> Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(Ni
>>>>>> mbusClient.java:90)
>>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>>>
>>>>>>
>>>>>> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <
>>>>>> jmenchaca@gobalto.com> wrote:
>>>>>>
>>>>>>>
>>>>>>> *Ideas?*
>>>>>>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>>>>>>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>>>>>>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>>>>>>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>>>>>>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>>>>>>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>>>>>>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>>>>>>> at clojure.core$some.invoke(core.clj:2570)
>>>>>>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>>>>>>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>>>>>>> at clojure.core$apply.invoke(core.clj:632)
>>>>>>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>>>>>>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>>>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>>>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>>>>>>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>>>>>>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>>>>>>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>>>>>>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>>>>>>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>>>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>>>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>>>>>>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>>>>>>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>>>>>>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>>>>>>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>>>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>>>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>>>>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>>>>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>>>>>>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>>>>>> at java.lang.Thread.run(Thread.java:745)
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> --
>>>>>>>
>>>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>> --
>>>>>>
>>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>>
>>>>>
>>>>
>>>>
>>>> --
>>>>
>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>
>>>
>>
>>
>> --
>>
>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>
>
--
是故勝兵先勝而後求戰，敗兵先戰而後求勝。
Re: NimbusLeaderNotFoundException problem?
Posted by Mostafa Gomaa <mg...@trendak.com>.
Can you us your zookeeper logs and config?
On Dec 13, 2016 2:32 AM, "Joaquin Menchaca" <jm...@gobalto.com> wrote:
> I am able to connect to the zookeeper:
>
> # printf "%s\n" $(echo ruok | nc zookeeper-stormcluster.gobalto.com 2181)
> imok
>
> But not always. I looked at the zookeeper server, t3.large now, and the
> CPU jumps around, sometimes 8%, sometimes 20%-35%, ocassionally 110%.
>
> I am not sure at this point where to look, it was working up until
> recently.
>
>
> On Mon, Dec 12, 2016 at 3:53 PM, Mostafa Gomaa <mg...@trendak.com> wrote:
>
>> Yes, your nimbus machine can't connect to zookeeper as per the log. Check
>> out that zookeeper is running and that there aren't any networking issues
>> between the two machines.
>>
>> On Dec 13, 2016 1:49 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>> wrote:
>>
>>> Storm.yaml (Storm 1.0.2):
>>>
>>> storm.zookeeper.servers:
>>> - "zookeeper-stormcluster.gobalto.com"
>>>
>>> nimbus.seeds: ["nimbus-stormcluster.gobalto.com", "
>>> nimbus2-stormcluster.gobalto.com"]
>>> nimbus.childopts: "-Xmx1024m -Djava.net.preferIPv4Stack=true"
>>> ui.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>>
>>> supervisor.childopts: "-Djava.net.preferIPv4Stack=true"
>>> worker.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>> supervisor.slots.ports:
>>> - 6700
>>> - 6701
>>> - 6702
>>> - 6703
>>>
>>> storm.local.dir: "/app/storm"
>>>
>>>
>>> Maybe my zookeeper is down or corrupt:
>>>
>>> 2016-12-12 23:46:50.628 o.a.s.s.o.a.c.ConnectionState [ERROR] Connection
>>> timed out for connection string (zookeeper-stormcluster.gobalto.com:2181)
>>> and timeout (15000) / elapsed (16327)
>>> org.apache.storm.shade.org.apache.curator.CuratorConnectionLossException:
>>> KeeperErrorCode = ConnectionLoss
>>> at org.apache.storm.shade.org.apache.curator.ConnectionState.ch
>>> eckTimeouts(ConnectionState.java:197)
>>> at org.apache.storm.shade.org.apache.curator.ConnectionState.ge
>>> tZooKeeper(ConnectionState.java:88)
>>> at org.apache.storm.shade.org.apache.curator.CuratorZookeeperCl
>>> ient.getZooKeeper(CuratorZookeeperClient.java:116)
>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Cur
>>> atorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:489)
>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:226)
>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:215)
>>> at org.apache.storm.shade.org.apache.curator.RetryLoop.callWith
>>> Retry(RetryLoop.java:108)
>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>> stsBuilderImpl.pathInForegroundStandard(ExistsBuilderImpl.java:212)
>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>> stsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:205)
>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:168)
>>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:39)
>>> at org.apache.storm.zookeeper$exists_node_QMARK_$fn__2090.invok
>>> e(zookeeper.clj:109)
>>> at org.apache.storm.zookeeper$exists_node_QMARK_.invoke(zookeep
>>> er.clj:106)
>>> at org.apache.storm.zookeeper$mkdirs.invoke(zookeeper.clj:126)
>>> at org.apache.storm.cluster_state.zookeeper_state_factory$_mkSt
>>> ate.invoke(zookeeper_state_factory.clj:29)
>>> at org.apache.storm.cluster_state.zookeeper_state_factory.mkState(Unknown
>>> Source)
>>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAcce
>>> ssorImpl.java:62)
>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMe
>>> thodAccessorImpl.java:43)
>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>> at org.apache.storm.cluster$mk_distributed_cluster_state.doInvo
>>> ke(cluster.clj:46)
>>> at clojure.lang.RestFn.invoke(RestFn.java:559)
>>> at org.apache.storm.cluster$mk_storm_cluster_state.doInvoke(clu
>>> ster.clj:250)
>>> at clojure.lang.RestFn.invoke(RestFn.java:486)
>>> at org.apache.storm.daemon.nimbus$nimbus_data.invoke(nimbus.clj:176)
>>> at org.apache.storm.daemon.nimbus$fn__7166$exec_fn__2466__auto_
>>> ___7167.invoke(nimbus.clj:1349)
>>> at clojure.lang.AFn.applyToHelper(AFn.java:156)
>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>> at clojure.core$apply.invoke(core.clj:630)
>>> at org.apache.storm.daemon.nimbus$fn__7166$service_handler__741
>>> 0.doInvoke(nimbus.clj:1346)
>>> at clojure.lang.RestFn.invoke(RestFn.java:421)
>>> at org.apache.storm.daemon.nimbus$launch_server_BANG_.invoke(ni
>>> mbus.clj:2193)
>>> at org.apache.storm.daemon.nimbus$_launch.invoke(nimbus.clj:2226)
>>> at org.apache.storm.daemon.nimbus$_main.invoke(nimbus.clj:2249)
>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>> at org.apache.storm.daemon.nimbus.main(Unknown Source)
>>> 2016-12-12 23:46:50.862 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:50.885 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>> connection established to zookeeper-stormcluster.gobalto
>>> .com/10.110.20.8:2181, initiating session
>>> 2016-12-12 23:46:50.886 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>> additional data from server sessionid 0x0, likely server has closed socket,
>>> closing socket connection and attempting reconnect
>>> 2016-12-12 23:46:51.578 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:51.602 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>> connection established to zookeeper-stormcluster.gobalto
>>> .com/10.110.20.8:2181, initiating session
>>> 2016-12-12 23:46:51.603 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>> additional data from server sessionid 0x0, likely server has closed socket,
>>> closing socket connection and attempting reconnect
>>> 2016-12-12 23:46:52.041 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:52.060 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>> connection established to zookeeper-stormcluster.gobalto
>>> .com/10.110.20.8:2181, initiating session
>>> 2016-12-12 23:46:52.061 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>>> additional data from server sessionid 0x0, likely server has closed socket,
>>> closing socket connection and attempting reconnect
>>> 2016-12-12 23:46:52.958 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:52.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket
>>> connection established to zookeeper-stormcluster.gobalto
>>> .com/10.110.20.8:2181, initiating session
>>> 2016-12-12 23:46:53.003 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181, unexpected
>>> error, closing socket connection and attempting reconnect
>>> java.io.IOException: Connection reset by peer
>>> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
>>> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
>>> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
>>> at sun.nio.ch.IOUtil.read(IOUtil.java:192)
>>> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doIO(ClientCnxnSocketNIO.java:68)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:366)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>> 2016-12-12 23:46:53.130 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:53.146 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server null, unexpected error, closing socket connection and attempting
>>> reconnect
>>> java.net.ConnectException: Connection refused
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>> .java:717)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>> 2016-12-12 23:46:53.463 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:53.474 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server null, unexpected error, closing socket connection and attempting
>>> reconnect
>>> java.net.ConnectException: Connection refused
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>> .java:717)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>> 2016-12-12 23:46:54.291 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:54.292 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server null, unexpected error, closing socket connection and attempting
>>> reconnect
>>> java.net.ConnectException: Connection refused
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>> .java:717)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>> 2016-12-12 23:46:54.440 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:54.454 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server null, unexpected error, closing socket connection and attempting
>>> reconnect
>>> java.net.ConnectException: Connection refused
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>> .java:717)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>> 2016-12-12 23:46:55.393 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:55.394 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server null, unexpected error, closing socket connection and attempting
>>> reconnect
>>> java.net.ConnectException: Connection refused
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>> .java:717)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>> 2016-12-12 23:46:55.741 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:55.763 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server null, unexpected error, closing socket connection and attempting
>>> reconnect
>>> java.net.ConnectException: Connection refused
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>> .java:717)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>> 2016-12-12 23:46:55.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>>> Will not attempt to authenticate using SASL (unknown error)
>>> 2016-12-12 23:46:55.969 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>>> server null, unexpected error, closing socket connection and attempting
>>> reconnect
>>> java.net.ConnectException: Connection refused
>>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>>> .java:717)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>>> hread.run(ClientCnxn.java:1081)
>>>
>>>
>>>
>>>
>>> On Mon, Dec 12, 2016 at 3:45 PM, Mostafa Gomaa <mg...@trendak.com>
>>> wrote:
>>>
>>>> Can you show us your config files as well as the nimbus log?
>>>>
>>>> On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>>>> wrote:
>>>>
>>>>> Also:
>>>>>
>>>>> $ storm list
>>>>> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
>>>>> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
>>>>> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
>>>>> -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
>>>>> -Dstorm.conf.file= -cp /usr/lib/apache/storm/1.0.2/li
>>>>> b/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-1
>>>>> .3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.jar:/
>>>>> usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/lib
>>>>> /apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.jar:/usr/lib
>>>>> /apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/usr/lib/apache/
>>>>> storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr/lib/apache/
>>>>> storm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache/storm/1.0.
>>>>> 2/lib/objenesis-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/
>>>>> log4j-core-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/servlet
>>>>> -api-2.5.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-over-slf
>>>>> 4j-1.6.6.jar:/usr/lib/apache/storm/1.0.2/lib/disruptor-3.3.
>>>>> 2.jar:/usr/lib/apache/storm/1.0.2/lib/reflectasm-1.10.1.jar:
>>>>> /usr/lib/apache/storm/1.0.2/lib/asm-5.0.3.jar:/usr/lib/apa
>>>>> che/storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
>>>>> org.apache.storm.command.list
>>>>> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>>> trying to get leader nimbus info from nimbus-stormcluster.gobalto.com.
>>>>> will retry with a different seed host.
>>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>>> java.net.ConnectException: Connection refused
>>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>> [clojure-1.7.0.jar:?]
>>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>>> java.net.ConnectException: Connection refused
>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> ... 8 more
>>>>> Caused by: java.net.ConnectException: Connection refused
>>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> ... 8 more
>>>>> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>>> trying to get leader nimbus info from nimbus2-stormcluster.gobalto.com.
>>>>> will retry with a different seed host.
>>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>>> java.net.ConnectException: Connection refused
>>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>> [clojure-1.7.0.jar:?]
>>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>> [storm-core-1.0.2.jar:1.0.2]
>>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>>> java.net.ConnectException: Connection refused
>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> ... 8 more
>>>>> Caused by: java.net.ConnectException: Connection refused
>>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>>> ~[?:1.8.0_91]
>>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>>> ... 8 more
>>>>> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
>>>>> Could not find leader nimbus from seed hosts ["
>>>>> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"].
>>>>> Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(Ni
>>>>> mbusClient.java:90)
>>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>>
>>>>>
>>>>> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <
>>>>> jmenchaca@gobalto.com> wrote:
>>>>>
>>>>>>
>>>>>> *Ideas?*
>>>>>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>>>>>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>>>>>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>>>>>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>>>>>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>>>>>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>>>>>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>>>>>> at clojure.core$some.invoke(core.clj:2570)
>>>>>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>>>>>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>>>>>> at clojure.core$apply.invoke(core.clj:632)
>>>>>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>>>>>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>>>>>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>>>>>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>>>>>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>>>>>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>>>>>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>>>>>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>>>>>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>>>>>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>>>>>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>>>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>>>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>>>>>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>>>>> at java.lang.Thread.run(Thread.java:745)
>>>>>>
>>>>>>
>>>>>>
>>>>>> --
>>>>>>
>>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>>
>>>>>
>>>>>
>>>>>
>>>>> --
>>>>>
>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>
>>>>
>>>
>>>
>>> --
>>>
>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>
>>
>
>
> --
>
> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>
Re: NimbusLeaderNotFoundException problem?
Posted by Joaquin Menchaca <jm...@gobalto.com>.
I am able to connect to the zookeeper:
# printf "%s\n" $(echo ruok | nc zookeeper-stormcluster.gobalto.com 2181)
imok
But not always. I looked at the zookeeper server, t3.large now, and the
CPU jumps around, sometimes 8%, sometimes 20%-35%, ocassionally 110%.
I am not sure at this point where to look, it was working up until recently.
On Mon, Dec 12, 2016 at 3:53 PM, Mostafa Gomaa <mg...@trendak.com> wrote:
> Yes, your nimbus machine can't connect to zookeeper as per the log. Check
> out that zookeeper is running and that there aren't any networking issues
> between the two machines.
>
> On Dec 13, 2016 1:49 AM, "Joaquin Menchaca" <jm...@gobalto.com> wrote:
>
>> Storm.yaml (Storm 1.0.2):
>>
>> storm.zookeeper.servers:
>> - "zookeeper-stormcluster.gobalto.com"
>>
>> nimbus.seeds: ["nimbus-stormcluster.gobalto.com", "
>> nimbus2-stormcluster.gobalto.com"]
>> nimbus.childopts: "-Xmx1024m -Djava.net.preferIPv4Stack=true"
>> ui.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>
>> supervisor.childopts: "-Djava.net.preferIPv4Stack=true"
>> worker.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>> supervisor.slots.ports:
>> - 6700
>> - 6701
>> - 6702
>> - 6703
>>
>> storm.local.dir: "/app/storm"
>>
>>
>> Maybe my zookeeper is down or corrupt:
>>
>> 2016-12-12 23:46:50.628 o.a.s.s.o.a.c.ConnectionState [ERROR] Connection
>> timed out for connection string (zookeeper-stormcluster.gobalto.com:2181)
>> and timeout (15000) / elapsed (16327)
>> org.apache.storm.shade.org.apache.curator.CuratorConnectionLossException:
>> KeeperErrorCode = ConnectionLoss
>> at org.apache.storm.shade.org.apache.curator.ConnectionState.
>> checkTimeouts(ConnectionState.java:197)
>> at org.apache.storm.shade.org.apache.curator.ConnectionState.
>> getZooKeeper(ConnectionState.java:88)
>> at org.apache.storm.shade.org.apache.curator.CuratorZookeeperCl
>> ient.getZooKeeper(CuratorZookeeperClient.java:116)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Cur
>> atorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:489)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:226)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:215)
>> at org.apache.storm.shade.org.apache.curator.RetryLoop.callWith
>> Retry(RetryLoop.java:108)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.pathInForegroundStandard(ExistsBuilderImpl.java:212)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:205)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:168)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:39)
>> at org.apache.storm.zookeeper$exists_node_QMARK_$fn__2090.invok
>> e(zookeeper.clj:109)
>> at org.apache.storm.zookeeper$exists_node_QMARK_.invoke(zookeep
>> er.clj:106)
>> at org.apache.storm.zookeeper$mkdirs.invoke(zookeeper.clj:126)
>> at org.apache.storm.cluster_state.zookeeper_state_factory$_
>> mkState.invoke(zookeeper_state_factory.clj:29)
>> at org.apache.storm.cluster_state.zookeeper_state_factory.mkState(Unknown
>> Source)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAcce
>> ssorImpl.java:62)
>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMe
>> thodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:498)
>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>> at org.apache.storm.cluster$mk_distributed_cluster_state.doInvo
>> ke(cluster.clj:46)
>> at clojure.lang.RestFn.invoke(RestFn.java:559)
>> at org.apache.storm.cluster$mk_storm_cluster_state.doInvoke(clu
>> ster.clj:250)
>> at clojure.lang.RestFn.invoke(RestFn.java:486)
>> at org.apache.storm.daemon.nimbus$nimbus_data.invoke(nimbus.clj:176)
>> at org.apache.storm.daemon.nimbus$fn__7166$exec_fn__2466__auto_
>> ___7167.invoke(nimbus.clj:1349)
>> at clojure.lang.AFn.applyToHelper(AFn.java:156)
>> at clojure.lang.AFn.applyTo(AFn.java:144)
>> at clojure.core$apply.invoke(core.clj:630)
>> at org.apache.storm.daemon.nimbus$fn__7166$service_handler__
>> 7410.doInvoke(nimbus.clj:1346)
>> at clojure.lang.RestFn.invoke(RestFn.java:421)
>> at org.apache.storm.daemon.nimbus$launch_server_BANG_.invoke(
>> nimbus.clj:2193)
>> at org.apache.storm.daemon.nimbus$_launch.invoke(nimbus.clj:2226)
>> at org.apache.storm.daemon.nimbus$_main.invoke(nimbus.clj:2249)
>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>> at clojure.lang.AFn.applyTo(AFn.java:144)
>> at org.apache.storm.daemon.nimbus.main(Unknown Source)
>> 2016-12-12 23:46:50.862 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:50.885 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:50.886 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>> additional data from server sessionid 0x0, likely server has closed socket,
>> closing socket connection and attempting reconnect
>> 2016-12-12 23:46:51.578 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:51.602 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:51.603 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>> additional data from server sessionid 0x0, likely server has closed socket,
>> closing socket connection and attempting reconnect
>> 2016-12-12 23:46:52.041 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:52.060 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:52.061 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>> additional data from server sessionid 0x0, likely server has closed socket,
>> closing socket connection and attempting reconnect
>> 2016-12-12 23:46:52.958 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:52.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:53.003 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181, unexpected
>> error, closing socket connection and attempting reconnect
>> java.io.IOException: Connection reset by peer
>> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
>> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
>> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
>> at sun.nio.ch.IOUtil.read(IOUtil.java:192)
>> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doIO(ClientCnxnSocketNIO.java:68)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:366)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:53.130 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:53.146 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:53.463 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:53.474 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:54.291 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:54.292 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:54.440 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:54.454 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:55.393 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:55.394 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:55.741 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:55.763 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:55.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:55.969 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>>
>>
>>
>>
>> On Mon, Dec 12, 2016 at 3:45 PM, Mostafa Gomaa <mg...@trendak.com>
>> wrote:
>>
>>> Can you show us your config files as well as the nimbus log?
>>>
>>> On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>>> wrote:
>>>
>>>> Also:
>>>>
>>>> $ storm list
>>>> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
>>>> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
>>>> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
>>>> -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
>>>> -Dstorm.conf.file= -cp /usr/lib/apache/storm/1.0.2/li
>>>> b/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-1
>>>> .3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.jar:/
>>>> usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/lib
>>>> /apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.jar:/usr/
>>>> lib/apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/usr/lib/
>>>> apache/storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr/lib/
>>>> apache/storm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache/storm/
>>>> 1.0.2/lib/objenesis-2.1.jar:/usr/lib/apache/storm/1.0.2/
>>>> lib/log4j-core-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/servl
>>>> et-api-2.5.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-over-
>>>> slf4j-1.6.6.jar:/usr/lib/apache/storm/1.0.2/lib/disruptor-3.
>>>> 3.2.jar:/usr/lib/apache/storm/1.0.2/lib/reflectasm-1.10.1.
>>>> jar:/usr/lib/apache/storm/1.0.2/lib/asm-5.0.3.jar:/usr/lib/
>>>> apache/storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
>>>> org.apache.storm.command.list
>>>> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>> trying to get leader nimbus info from nimbus-stormcluster.gobalto.com.
>>>> will retry with a different seed host.
>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>> [clojure-1.7.0.jar:?]
>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> Caused by: java.net.ConnectException: Connection refused
>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>> ~[?:1.8.0_91]
>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>> ~[?:1.8.0_91]
>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>> trying to get leader nimbus info from nimbus2-stormcluster.gobalto.com.
>>>> will retry with a different seed host.
>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>> [clojure-1.7.0.jar:?]
>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> Caused by: java.net.ConnectException: Connection refused
>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>> ~[?:1.8.0_91]
>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>> ~[?:1.8.0_91]
>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
>>>> Could not find leader nimbus from seed hosts ["
>>>> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"].
>>>> Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(Ni
>>>> mbusClient.java:90)
>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>
>>>>
>>>> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <
>>>> jmenchaca@gobalto.com> wrote:
>>>>
>>>>>
>>>>> *Ideas?*
>>>>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>>>>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>>>>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>>>>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>>>>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>>>>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>>>>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>>>>> at clojure.core$some.invoke(core.clj:2570)
>>>>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>>>>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>>>>> at clojure.core$apply.invoke(core.clj:632)
>>>>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>>>>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>>>>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>>>>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>>>>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>>>>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>>>>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>>>>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>>>>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>>>>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>>>>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>>>>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>>>> at java.lang.Thread.run(Thread.java:745)
>>>>>
>>>>>
>>>>>
>>>>> --
>>>>>
>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>
>>>>
>>>>
>>>>
>>>> --
>>>>
>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>
>>>
>>
>>
>> --
>>
>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>
>
--
是故勝兵先勝而後求戰，敗兵先戰而後求勝。
Re: NimbusLeaderNotFoundException problem?
Posted by Joaquin Menchaca <jm...@gobalto.com>.
Yeha, I think zookeeper was t2.micro, thrashing from 75% - 95% of cpu.
Upgrading it, hopefully that will make everything work...
On Mon, Dec 12, 2016 at 3:53 PM, Mostafa Gomaa <mg...@trendak.com> wrote:
> Yes, your nimbus machine can't connect to zookeeper as per the log. Check
> out that zookeeper is running and that there aren't any networking issues
> between the two machines.
>
> On Dec 13, 2016 1:49 AM, "Joaquin Menchaca" <jm...@gobalto.com> wrote:
>
>> Storm.yaml (Storm 1.0.2):
>>
>> storm.zookeeper.servers:
>> - "zookeeper-stormcluster.gobalto.com"
>>
>> nimbus.seeds: ["nimbus-stormcluster.gobalto.com", "
>> nimbus2-stormcluster.gobalto.com"]
>> nimbus.childopts: "-Xmx1024m -Djava.net.preferIPv4Stack=true"
>> ui.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>>
>> supervisor.childopts: "-Djava.net.preferIPv4Stack=true"
>> worker.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>> supervisor.slots.ports:
>> - 6700
>> - 6701
>> - 6702
>> - 6703
>>
>> storm.local.dir: "/app/storm"
>>
>>
>> Maybe my zookeeper is down or corrupt:
>>
>> 2016-12-12 23:46:50.628 o.a.s.s.o.a.c.ConnectionState [ERROR] Connection
>> timed out for connection string (zookeeper-stormcluster.gobalto.com:2181)
>> and timeout (15000) / elapsed (16327)
>> org.apache.storm.shade.org.apache.curator.CuratorConnectionLossException:
>> KeeperErrorCode = ConnectionLoss
>> at org.apache.storm.shade.org.apache.curator.ConnectionState.
>> checkTimeouts(ConnectionState.java:197)
>> at org.apache.storm.shade.org.apache.curator.ConnectionState.
>> getZooKeeper(ConnectionState.java:88)
>> at org.apache.storm.shade.org.apache.curator.CuratorZookeeperCl
>> ient.getZooKeeper(CuratorZookeeperClient.java:116)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Cur
>> atorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:489)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:226)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl$3.call(ExistsBuilderImpl.java:215)
>> at org.apache.storm.shade.org.apache.curator.RetryLoop.callWith
>> Retry(RetryLoop.java:108)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.pathInForegroundStandard(ExistsBuilderImpl.java:212)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:205)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:168)
>> at org.apache.storm.shade.org.apache.curator.framework.imps.Exi
>> stsBuilderImpl.forPath(ExistsBuilderImpl.java:39)
>> at org.apache.storm.zookeeper$exists_node_QMARK_$fn__2090.invok
>> e(zookeeper.clj:109)
>> at org.apache.storm.zookeeper$exists_node_QMARK_.invoke(zookeep
>> er.clj:106)
>> at org.apache.storm.zookeeper$mkdirs.invoke(zookeeper.clj:126)
>> at org.apache.storm.cluster_state.zookeeper_state_factory$_
>> mkState.invoke(zookeeper_state_factory.clj:29)
>> at org.apache.storm.cluster_state.zookeeper_state_factory.mkState(Unknown
>> Source)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAcce
>> ssorImpl.java:62)
>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMe
>> thodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:498)
>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>> at org.apache.storm.cluster$mk_distributed_cluster_state.doInvo
>> ke(cluster.clj:46)
>> at clojure.lang.RestFn.invoke(RestFn.java:559)
>> at org.apache.storm.cluster$mk_storm_cluster_state.doInvoke(clu
>> ster.clj:250)
>> at clojure.lang.RestFn.invoke(RestFn.java:486)
>> at org.apache.storm.daemon.nimbus$nimbus_data.invoke(nimbus.clj:176)
>> at org.apache.storm.daemon.nimbus$fn__7166$exec_fn__2466__auto_
>> ___7167.invoke(nimbus.clj:1349)
>> at clojure.lang.AFn.applyToHelper(AFn.java:156)
>> at clojure.lang.AFn.applyTo(AFn.java:144)
>> at clojure.core$apply.invoke(core.clj:630)
>> at org.apache.storm.daemon.nimbus$fn__7166$service_handler__
>> 7410.doInvoke(nimbus.clj:1346)
>> at clojure.lang.RestFn.invoke(RestFn.java:421)
>> at org.apache.storm.daemon.nimbus$launch_server_BANG_.invoke(
>> nimbus.clj:2193)
>> at org.apache.storm.daemon.nimbus$_launch.invoke(nimbus.clj:2226)
>> at org.apache.storm.daemon.nimbus$_main.invoke(nimbus.clj:2249)
>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>> at clojure.lang.AFn.applyTo(AFn.java:144)
>> at org.apache.storm.daemon.nimbus.main(Unknown Source)
>> 2016-12-12 23:46:50.862 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:50.885 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:50.886 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>> additional data from server sessionid 0x0, likely server has closed socket,
>> closing socket connection and attempting reconnect
>> 2016-12-12 23:46:51.578 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:51.602 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:51.603 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>> additional data from server sessionid 0x0, likely server has closed socket,
>> closing socket connection and attempting reconnect
>> 2016-12-12 23:46:52.041 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:52.060 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:52.061 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
>> additional data from server sessionid 0x0, likely server has closed socket,
>> closing socket connection and attempting reconnect
>> 2016-12-12 23:46:52.958 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:52.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
>> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
>> initiating session
>> 2016-12-12 23:46:53.003 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181, unexpected
>> error, closing socket connection and attempting reconnect
>> java.io.IOException: Connection reset by peer
>> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
>> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
>> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
>> at sun.nio.ch.IOUtil.read(IOUtil.java:192)
>> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doIO(ClientCnxnSocketNIO.java:68)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:366)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:53.130 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:53.146 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:53.463 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:53.474 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:54.291 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:54.292 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:54.440 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:54.454 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:55.393 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:55.394 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:55.741 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:55.763 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>> 2016-12-12 23:46:55.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
>> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
>> Will not attempt to authenticate using SASL (unknown error)
>> 2016-12-12 23:46:55.969 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
>> server null, unexpected error, closing socket connection and attempting
>> reconnect
>> java.net.ConnectException: Connection refused
>> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
>> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl
>> .java:717)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocket
>> NIO.doTransport(ClientCnxnSocketNIO.java:361)
>> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendT
>> hread.run(ClientCnxn.java:1081)
>>
>>
>>
>>
>> On Mon, Dec 12, 2016 at 3:45 PM, Mostafa Gomaa <mg...@trendak.com>
>> wrote:
>>
>>> Can you show us your config files as well as the nimbus log?
>>>
>>> On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>>> wrote:
>>>
>>>> Also:
>>>>
>>>> $ storm list
>>>> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
>>>> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
>>>> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
>>>> -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
>>>> -Dstorm.conf.file= -cp /usr/lib/apache/storm/1.0.2/li
>>>> b/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-1
>>>> .3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.jar:/
>>>> usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/lib
>>>> /apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.jar:/usr/
>>>> lib/apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/usr/lib/
>>>> apache/storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr/lib/
>>>> apache/storm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache/storm/
>>>> 1.0.2/lib/objenesis-2.1.jar:/usr/lib/apache/storm/1.0.2/
>>>> lib/log4j-core-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/servl
>>>> et-api-2.5.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-over-
>>>> slf4j-1.6.6.jar:/usr/lib/apache/storm/1.0.2/lib/disruptor-3.
>>>> 3.2.jar:/usr/lib/apache/storm/1.0.2/lib/reflectasm-1.10.1.
>>>> jar:/usr/lib/apache/storm/1.0.2/lib/asm-5.0.3.jar:/usr/lib/
>>>> apache/storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
>>>> org.apache.storm.command.list
>>>> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>> trying to get leader nimbus info from nimbus-stormcluster.gobalto.com.
>>>> will retry with a different seed host.
>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>> [clojure-1.7.0.jar:?]
>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> Caused by: java.net.ConnectException: Connection refused
>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>> ~[?:1.8.0_91]
>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>> ~[?:1.8.0_91]
>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>>> trying to get leader nimbus info from nimbus2-stormcluster.gobalto.com.
>>>> will retry with a different seed host.
>>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>> [clojure-1.7.0.jar:?]
>>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>> [storm-core-1.0.2.jar:1.0.2]
>>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>>> java.net.ConnectException: Connection refused
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> Caused by: java.net.ConnectException: Connection refused
>>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>>> ~[?:1.8.0_91]
>>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>>> ~[?:1.8.0_91]
>>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>>> ~[?:1.8.0_91]
>>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>>> ~[storm-core-1.0.2.jar:1.0.2]
>>>> ... 8 more
>>>> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
>>>> Could not find leader nimbus from seed hosts ["
>>>> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"].
>>>> Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(Ni
>>>> mbusClient.java:90)
>>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>>> at org.apache.storm.command.list.main(Unknown Source)
>>>>
>>>>
>>>> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <
>>>> jmenchaca@gobalto.com> wrote:
>>>>
>>>>>
>>>>> *Ideas?*
>>>>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>>>>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>>>>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>>>>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>>>>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>>>>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>>>>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>>>>> at clojure.core$some.invoke(core.clj:2570)
>>>>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>>>>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>>>>> at clojure.core$apply.invoke(core.clj:632)
>>>>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>>>>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>>>>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>>>>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>>>>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>>>>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>>>>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>>>>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>>>>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>>>>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>>>>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>>>>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>>>> at java.lang.Thread.run(Thread.java:745)
>>>>>
>>>>>
>>>>>
>>>>> --
>>>>>
>>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>>
>>>>
>>>>
>>>>
>>>> --
>>>>
>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>
>>>
>>
>>
>> --
>>
>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>
>
--
是故勝兵先勝而後求戰，敗兵先戰而後求勝。
Re: NimbusLeaderNotFoundException problem?
Posted by Mostafa Gomaa <mg...@trendak.com>.
Yes, your nimbus machine can't connect to zookeeper as per the log. Check
out that zookeeper is running and that there aren't any networking issues
between the two machines.
On Dec 13, 2016 1:49 AM, "Joaquin Menchaca" <jm...@gobalto.com> wrote:
> Storm.yaml (Storm 1.0.2):
>
> storm.zookeeper.servers:
> - "zookeeper-stormcluster.gobalto.com"
>
> nimbus.seeds: ["nimbus-stormcluster.gobalto.com", "
> nimbus2-stormcluster.gobalto.com"]
> nimbus.childopts: "-Xmx1024m -Djava.net.preferIPv4Stack=true"
> ui.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
>
> supervisor.childopts: "-Djava.net.preferIPv4Stack=true"
> worker.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
> supervisor.slots.ports:
> - 6700
> - 6701
> - 6702
> - 6703
>
> storm.local.dir: "/app/storm"
>
>
> Maybe my zookeeper is down or corrupt:
>
> 2016-12-12 23:46:50.628 o.a.s.s.o.a.c.ConnectionState [ERROR] Connection
> timed out for connection string (zookeeper-stormcluster.gobalto.com:2181)
> and timeout (15000) / elapsed (16327)
> org.apache.storm.shade.org.apache.curator.CuratorConnectionLossException:
> KeeperErrorCode = ConnectionLoss
> at org.apache.storm.shade.org.apache.curator.
> ConnectionState.checkTimeouts(ConnectionState.java:197)
> at org.apache.storm.shade.org.apache.curator.
> ConnectionState.getZooKeeper(ConnectionState.java:88)
> at org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.
> getZooKeeper(CuratorZookeeperClient.java:116)
> at org.apache.storm.shade.org.apache.curator.framework.imps.
> CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:489)
> at org.apache.storm.shade.org.apache.curator.framework.imps.
> ExistsBuilderImpl$3.call(ExistsBuilderImpl.java:226)
> at org.apache.storm.shade.org.apache.curator.framework.imps.
> ExistsBuilderImpl$3.call(ExistsBuilderImpl.java:215)
> at org.apache.storm.shade.org.apache.curator.RetryLoop.
> callWithRetry(RetryLoop.java:108)
> at org.apache.storm.shade.org.apache.curator.framework.imps.
> ExistsBuilderImpl.pathInForegroundStandard(ExistsBuilderImpl.java:212)
> at org.apache.storm.shade.org.apache.curator.framework.imps.
> ExistsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:205)
> at org.apache.storm.shade.org.apache.curator.framework.imps.
> ExistsBuilderImpl.forPath(ExistsBuilderImpl.java:168)
> at org.apache.storm.shade.org.apache.curator.framework.imps.
> ExistsBuilderImpl.forPath(ExistsBuilderImpl.java:39)
> at org.apache.storm.zookeeper$exists_node_QMARK_$fn__2090.
> invoke(zookeeper.clj:109)
> at org.apache.storm.zookeeper$exists_node_QMARK_.invoke(
> zookeeper.clj:106)
> at org.apache.storm.zookeeper$mkdirs.invoke(zookeeper.clj:126)
> at org.apache.storm.cluster_state.zookeeper_state_factory$
> _mkState.invoke(zookeeper_state_factory.clj:29)
> at org.apache.storm.cluster_state.zookeeper_state_factory.mkState(Unknown
> Source)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
> at org.apache.storm.cluster$mk_distributed_cluster_state.
> doInvoke(cluster.clj:46)
> at clojure.lang.RestFn.invoke(RestFn.java:559)
> at org.apache.storm.cluster$mk_storm_cluster_state.doInvoke(
> cluster.clj:250)
> at clojure.lang.RestFn.invoke(RestFn.java:486)
> at org.apache.storm.daemon.nimbus$nimbus_data.invoke(nimbus.clj:176)
> at org.apache.storm.daemon.nimbus$fn__7166$exec_fn__2466_
> _auto____7167.invoke(nimbus.clj:1349)
> at clojure.lang.AFn.applyToHelper(AFn.java:156)
> at clojure.lang.AFn.applyTo(AFn.java:144)
> at clojure.core$apply.invoke(core.clj:630)
> at org.apache.storm.daemon.nimbus$fn__7166$service_
> handler__7410.doInvoke(nimbus.clj:1346)
> at clojure.lang.RestFn.invoke(RestFn.java:421)
> at org.apache.storm.daemon.nimbus$launch_server_BANG_.
> invoke(nimbus.clj:2193)
> at org.apache.storm.daemon.nimbus$_launch.invoke(nimbus.clj:2226)
> at org.apache.storm.daemon.nimbus$_main.invoke(nimbus.clj:2249)
> at clojure.lang.AFn.applyToHelper(AFn.java:152)
> at clojure.lang.AFn.applyTo(AFn.java:144)
> at org.apache.storm.daemon.nimbus.main(Unknown Source)
> 2016-12-12 23:46:50.862 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:50.885 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
> initiating session
> 2016-12-12 23:46:50.886 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
> additional data from server sessionid 0x0, likely server has closed socket,
> closing socket connection and attempting reconnect
> 2016-12-12 23:46:51.578 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:51.602 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
> initiating session
> 2016-12-12 23:46:51.603 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
> additional data from server sessionid 0x0, likely server has closed socket,
> closing socket connection and attempting reconnect
> 2016-12-12 23:46:52.041 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:52.060 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
> initiating session
> 2016-12-12 23:46:52.061 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
> additional data from server sessionid 0x0, likely server has closed socket,
> closing socket connection and attempting reconnect
> 2016-12-12 23:46:52.958 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:52.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
> established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
> initiating session
> 2016-12-12 23:46:53.003 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181, unexpected
> error, closing socket connection and attempting reconnect
> java.io.IOException: Connection reset by peer
> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
> at sun.nio.ch.IOUtil.read(IOUtil.java:192)
> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
> at org.apache.storm.shade.org.apache.zookeeper.
> ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:68)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:366)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
> 2016-12-12 23:46:53.130 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:53.146 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server null, unexpected error, closing socket connection and attempting
> reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> SocketChannelImpl.java:717)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:361)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
> 2016-12-12 23:46:53.463 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:53.474 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server null, unexpected error, closing socket connection and attempting
> reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> SocketChannelImpl.java:717)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:361)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
> 2016-12-12 23:46:54.291 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:54.292 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server null, unexpected error, closing socket connection and attempting
> reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> SocketChannelImpl.java:717)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:361)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
> 2016-12-12 23:46:54.440 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:54.454 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server null, unexpected error, closing socket connection and attempting
> reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> SocketChannelImpl.java:717)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:361)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
> 2016-12-12 23:46:55.393 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:55.394 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server null, unexpected error, closing socket connection and attempting
> reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> SocketChannelImpl.java:717)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:361)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
> 2016-12-12 23:46:55.741 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:55.763 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server null, unexpected error, closing socket connection and attempting
> reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> SocketChannelImpl.java:717)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:361)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
> 2016-12-12 23:46:55.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
> connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
> Will not attempt to authenticate using SASL (unknown error)
> 2016-12-12 23:46:55.969 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
> server null, unexpected error, closing socket connection and attempting
> reconnect
> java.net.ConnectException: Connection refused
> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
> at sun.nio.ch.SocketChannelImpl.finishConnect(
> SocketChannelImpl.java:717)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.
> doTransport(ClientCnxnSocketNIO.java:361)
> at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$
> SendThread.run(ClientCnxn.java:1081)
>
>
>
>
> On Mon, Dec 12, 2016 at 3:45 PM, Mostafa Gomaa <mg...@trendak.com> wrote:
>
>> Can you show us your config files as well as the nimbus log?
>>
>> On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com>
>> wrote:
>>
>>> Also:
>>>
>>> $ storm list
>>> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
>>> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
>>> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
>>> -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
>>> -Dstorm.conf.file= -cp /usr/lib/apache/storm/1.0.2/li
>>> b/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-1
>>> .3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.jar:/
>>> usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/
>>> lib/apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.jar:/
>>> usr/lib/apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/usr/
>>> lib/apache/storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr/
>>> lib/apache/storm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache/
>>> storm/1.0.2/lib/objenesis-2.1.jar:/usr/lib/apache/storm/1.0.
>>> 2/lib/log4j-core-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/
>>> servlet-api-2.5.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-
>>> over-slf4j-1.6.6.jar:/usr/lib/apache/storm/1.0.2/lib/
>>> disruptor-3.3.2.jar:/usr/lib/apache/storm/1.0.2/lib/reflect
>>> asm-1.10.1.jar:/usr/lib/apache/storm/1.0.2/lib/asm-5.
>>> 0.3.jar:/usr/lib/apache/storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
>>> org.apache.storm.command.list
>>> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>> trying to get leader nimbus info from nimbus-stormcluster.gobalto.com.
>>> will retry with a different seed host.
>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>> java.net.ConnectException: Connection refused
>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>> [storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>> [storm-core-1.0.2.jar:1.0.2]
>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>> [clojure-1.7.0.jar:?]
>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>> at org.apache.storm.command.list.main(Unknown Source)
>>> [storm-core-1.0.2.jar:1.0.2]
>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>> java.net.ConnectException: Connection refused
>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> ... 8 more
>>> Caused by: java.net.ConnectException: Connection refused
>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>> ~[?:1.8.0_91]
>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>> ~[?:1.8.0_91]
>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>> ~[?:1.8.0_91]
>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>> ~[?:1.8.0_91]
>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>> ~[?:1.8.0_91]
>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> ... 8 more
>>> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while
>>> trying to get leader nimbus info from nimbus2-stormcluster.gobalto.com.
>>> will retry with a different seed host.
>>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>>> java.net.ConnectException: Connection refused
>>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>>> [storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>> [storm-core-1.0.2.jar:1.0.2]
>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>> [clojure-1.7.0.jar:?]
>>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>>> at org.apache.storm.command.list.main(Unknown Source)
>>> [storm-core-1.0.2.jar:1.0.2]
>>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>>> java.net.ConnectException: Connection refused
>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> ... 8 more
>>> Caused by: java.net.ConnectException: Connection refused
>>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>>> ~[?:1.8.0_91]
>>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>>> ~[?:1.8.0_91]
>>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>>> ~[?:1.8.0_91]
>>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>>> ~[?:1.8.0_91]
>>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>>> ~[?:1.8.0_91]
>>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>>> ~[storm-core-1.0.2.jar:1.0.2]
>>> ... 8 more
>>> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
>>> Could not find leader nimbus from seed hosts ["
>>> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"].
>>> Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(Ni
>>> mbusClient.java:90)
>>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>>> at clojure.lang.AFn.applyTo(AFn.java:144)
>>> at org.apache.storm.command.list.main(Unknown Source)
>>>
>>>
>>> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <jmenchaca@gobalto.com
>>> > wrote:
>>>
>>>>
>>>> *Ideas?*
>>>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>>>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>>>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>>>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>>>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>>>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>>>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>>>> at clojure.core$some.invoke(core.clj:2570)
>>>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>>>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>>>> at clojure.core$apply.invoke(core.clj:632)
>>>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>>>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>>>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>>>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>>>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>>>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>>>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>>>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>>>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>>>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>>>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>>>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>>>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>>>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>>> at java.lang.Thread.run(Thread.java:745)
>>>>
>>>>
>>>>
>>>> --
>>>>
>>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>>
>>>
>>>
>>>
>>> --
>>>
>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>
>>
>
>
> --
>
> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>
Re: NimbusLeaderNotFoundException problem?
Posted by Joaquin Menchaca <jm...@gobalto.com>.
Storm.yaml (Storm 1.0.2):
storm.zookeeper.servers:
- "zookeeper-stormcluster.gobalto.com"
nimbus.seeds: ["nimbus-stormcluster.gobalto.com", "
nimbus2-stormcluster.gobalto.com"]
nimbus.childopts: "-Xmx1024m -Djava.net.preferIPv4Stack=true"
ui.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
supervisor.childopts: "-Djava.net.preferIPv4Stack=true"
worker.childopts: "-Xmx768m -Djava.net.preferIPv4Stack=true"
supervisor.slots.ports:
- 6700
- 6701
- 6702
- 6703
storm.local.dir: "/app/storm"
Maybe my zookeeper is down or corrupt:
2016-12-12 23:46:50.628 o.a.s.s.o.a.c.ConnectionState [ERROR] Connection
timed out for connection string (zookeeper-stormcluster.gobalto.com:2181)
and timeout (15000) / elapsed (16327)
org.apache.storm.shade.org.apache.curator.CuratorConnectionLossException:
KeeperErrorCode = ConnectionLoss
at
org.apache.storm.shade.org.apache.curator.ConnectionState.checkTimeouts(ConnectionState.java:197)
at
org.apache.storm.shade.org.apache.curator.ConnectionState.getZooKeeper(ConnectionState.java:88)
at
org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:116)
at
org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:489)
at
org.apache.storm.shade.org.apache.curator.framework.imps.ExistsBuilderImpl$3.call(ExistsBuilderImpl.java:226)
at
org.apache.storm.shade.org.apache.curator.framework.imps.ExistsBuilderImpl$3.call(ExistsBuilderImpl.java:215)
at
org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:108)
at
org.apache.storm.shade.org.apache.curator.framework.imps.ExistsBuilderImpl.pathInForegroundStandard(ExistsBuilderImpl.java:212)
at
org.apache.storm.shade.org.apache.curator.framework.imps.ExistsBuilderImpl.pathInForeground(ExistsBuilderImpl.java:205)
at
org.apache.storm.shade.org.apache.curator.framework.imps.ExistsBuilderImpl.forPath(ExistsBuilderImpl.java:168)
at
org.apache.storm.shade.org.apache.curator.framework.imps.ExistsBuilderImpl.forPath(ExistsBuilderImpl.java:39)
at
org.apache.storm.zookeeper$exists_node_QMARK_$fn__2090.invoke(zookeeper.clj:109)
at
org.apache.storm.zookeeper$exists_node_QMARK_.invoke(zookeeper.clj:106)
at org.apache.storm.zookeeper$mkdirs.invoke(zookeeper.clj:126)
at
org.apache.storm.cluster_state.zookeeper_state_factory$_mkState.invoke(zookeeper_state_factory.clj:29)
at
org.apache.storm.cluster_state.zookeeper_state_factory.mkState(Unknown
Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
at
org.apache.storm.cluster$mk_distributed_cluster_state.doInvoke(cluster.clj:46)
at clojure.lang.RestFn.invoke(RestFn.java:559)
at
org.apache.storm.cluster$mk_storm_cluster_state.doInvoke(cluster.clj:250)
at clojure.lang.RestFn.invoke(RestFn.java:486)
at org.apache.storm.daemon.nimbus$nimbus_data.invoke(nimbus.clj:176)
at
org.apache.storm.daemon.nimbus$fn__7166$exec_fn__2466__auto____7167.invoke(nimbus.clj:1349)
at clojure.lang.AFn.applyToHelper(AFn.java:156)
at clojure.lang.AFn.applyTo(AFn.java:144)
at clojure.core$apply.invoke(core.clj:630)
at
org.apache.storm.daemon.nimbus$fn__7166$service_handler__7410.doInvoke(nimbus.clj:1346)
at clojure.lang.RestFn.invoke(RestFn.java:421)
at
org.apache.storm.daemon.nimbus$launch_server_BANG_.invoke(nimbus.clj:2193)
at org.apache.storm.daemon.nimbus$_launch.invoke(nimbus.clj:2226)
at org.apache.storm.daemon.nimbus$_main.invoke(nimbus.clj:2249)
at clojure.lang.AFn.applyToHelper(AFn.java:152)
at clojure.lang.AFn.applyTo(AFn.java:144)
at org.apache.storm.daemon.nimbus.main(Unknown Source)
2016-12-12 23:46:50.862 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:50.885 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
initiating session
2016-12-12 23:46:50.886 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
additional data from server sessionid 0x0, likely server has closed socket,
closing socket connection and attempting reconnect
2016-12-12 23:46:51.578 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:51.602 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
initiating session
2016-12-12 23:46:51.603 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
additional data from server sessionid 0x0, likely server has closed socket,
closing socket connection and attempting reconnect
2016-12-12 23:46:52.041 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:52.060 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
initiating session
2016-12-12 23:46:52.061 o.a.s.s.o.a.z.ClientCnxn [INFO] Unable to read
additional data from server sessionid 0x0, likely server has closed socket,
closing socket connection and attempting reconnect
2016-12-12 23:46:52.958 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:52.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Socket connection
established to zookeeper-stormcluster.gobalto.com/10.110.20.8:2181,
initiating session
2016-12-12 23:46:53.003 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181, unexpected
error, closing socket connection and attempting reconnect
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:192)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:68)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:366)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2016-12-12 23:46:53.130 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:53.146 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server null, unexpected error, closing socket connection and attempting
reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2016-12-12 23:46:53.463 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:53.474 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server null, unexpected error, closing socket connection and attempting
reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2016-12-12 23:46:54.291 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:54.292 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server null, unexpected error, closing socket connection and attempting
reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2016-12-12 23:46:54.440 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:54.454 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server null, unexpected error, closing socket connection and attempting
reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2016-12-12 23:46:55.393 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:55.394 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server null, unexpected error, closing socket connection and attempting
reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2016-12-12 23:46:55.741 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:55.763 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server null, unexpected error, closing socket connection and attempting
reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
2016-12-12 23:46:55.959 o.a.s.s.o.a.z.ClientCnxn [INFO] Opening socket
connection to server zookeeper-stormcluster.gobalto.com/10.110.20.8:2181.
Will not attempt to authenticate using SASL (unknown error)
2016-12-12 23:46:55.969 o.a.s.s.o.a.z.ClientCnxn [WARN] Session 0x0 for
server null, unexpected error, closing socket connection and attempting
reconnect
java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at
sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
at
org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
On Mon, Dec 12, 2016 at 3:45 PM, Mostafa Gomaa <mg...@trendak.com> wrote:
> Can you show us your config files as well as the nimbus log?
>
> On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com> wrote:
>
>> Also:
>>
>> $ storm list
>> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
>> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
>> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
>> -Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
>> -Dstorm.conf.file= -cp /usr/lib/apache/storm/1.0.2/li
>> b/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-
>> 1.3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.
>> jar:/usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/
>> usr/lib/apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.
>> jar:/usr/lib/apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/
>> usr/lib/apache/storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr
>> /lib/apache/storm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache
>> /storm/1.0.2/lib/objenesis-2.1.jar:/usr/lib/apache/storm/1.
>> 0.2/lib/log4j-core-2.1.jar:/usr/lib/apache/storm/1.0.2/
>> lib/servlet-api-2.5.jar:/usr/lib/apache/storm/1.0.2/lib/
>> log4j-over-slf4j-1.6.6.jar:/usr/lib/apache/storm/1.0.2/
>> lib/disruptor-3.3.2.jar:/usr/lib/apache/storm/1.0.2/lib/
>> reflectasm-1.10.1.jar:/usr/lib/apache/storm/1.0.2/lib/
>> asm-5.0.3.jar:/usr/lib/apache/storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
>> org.apache.storm.command.list
>> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while trying
>> to get leader nimbus info from nimbus-stormcluster.gobalto.com. will
>> retry with a different seed host.
>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>> java.net.ConnectException: Connection refused
>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>> [storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>> [storm-core-1.0.2.jar:1.0.2]
>> at clojure.lang.AFn.applyToHelper(AFn.java:152) [clojure-1.7.0.jar:?]
>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>> at org.apache.storm.command.list.main(Unknown Source)
>> [storm-core-1.0.2.jar:1.0.2]
>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>> java.net.ConnectException: Connection refused
>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> ... 8 more
>> Caused by: java.net.ConnectException: Connection refused
>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>> ~[?:1.8.0_91]
>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>> ~[?:1.8.0_91]
>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>> ~[?:1.8.0_91]
>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>> ~[?:1.8.0_91]
>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>> ~[?:1.8.0_91]
>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> ... 8 more
>> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while trying
>> to get leader nimbus info from nimbus2-stormcluster.gobalto.com. will
>> retry with a different seed host.
>> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
>> java.net.ConnectException: Connection refused
>> at org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
>> [storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>> [storm-core-1.0.2.jar:1.0.2]
>> at clojure.lang.AFn.applyToHelper(AFn.java:152) [clojure-1.7.0.jar:?]
>> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
>> at org.apache.storm.command.list.main(Unknown Source)
>> [storm-core-1.0.2.jar:1.0.2]
>> Caused by: org.apache.storm.thrift.transport.TTransportException:
>> java.net.ConnectException: Connection refused
>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> ... 8 more
>> Caused by: java.net.ConnectException: Connection refused
>> at java.net.PlainSocketImpl.socketConnect(Native Method)
>> ~[?:1.8.0_91]
>> at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
>> ~[?:1.8.0_91]
>> at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
>> ~[?:1.8.0_91]
>> at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
>> ~[?:1.8.0_91]
>> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
>> ~[?:1.8.0_91]
>> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
>> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.SimpleTransportPlugin.connect
>> (SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
>> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
>> ~[storm-core-1.0.2.jar:1.0.2]
>> ... 8 more
>> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
>> Could not find leader nimbus from seed hosts ["
>> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"].
>> Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(Ni
>> mbusClient.java:90)
>> at org.apache.storm.command.list$_main.invoke(list.clj:22)
>> at clojure.lang.AFn.applyToHelper(AFn.java:152)
>> at clojure.lang.AFn.applyTo(AFn.java:144)
>> at org.apache.storm.command.list.main(Unknown Source)
>>
>>
>> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <jm...@gobalto.com>
>> wrote:
>>
>>>
>>> *Ideas?*
>>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>>> at clojure.core$some.invoke(core.clj:2570)
>>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>>> at clojure.core$apply.invoke(core.clj:632)
>>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>> at java.lang.reflect.Method.invoke(Method.java:498)
>>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>>> at java.lang.Thread.run(Thread.java:745)
>>>
>>>
>>>
>>> --
>>>
>>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>>
>>
>>
>>
>> --
>>
>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>
>
--
是故勝兵先勝而後求戰，敗兵先戰而後求勝。
Re: NimbusLeaderNotFoundException problem?
Posted by Mostafa Gomaa <mg...@trendak.com>.
Can you show us your config files as well as the nimbus log?
On Dec 13, 2016 1:44 AM, "Joaquin Menchaca" <jm...@gobalto.com> wrote:
> Also:
>
> $ storm list
> Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
> -Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
> -Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs -Djava.library.path=/usr/
> local/lib:/opt/local/lib:/usr/lib -Dstorm.conf.file= -cp
> /usr/lib/apache/storm/1.0.2/lib/clojure-1.7.0.jar:/usr/
> lib/apache/storm/1.0.2/lib/minlog-1.3.0.jar:/usr/lib/
> apache/storm/1.0.2/lib/log4j-api-2.1.jar:/usr/lib/apache/
> storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/lib/apache/storm/
> 1.0.2/lib/storm-rename-hack-1.0.2.jar:/usr/lib/apache/storm/
> 1.0.2/lib/slf4j-api-1.7.7.jar:/usr/lib/apache/storm/1.0.2/
> lib/log4j-slf4j-impl-2.1.jar:/usr/lib/apache/storm/1.0.2/
> lib/kryo-3.0.3.jar:/usr/lib/apache/storm/1.0.2/lib/
> objenesis-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-
> core-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/servlet-api-2.
> 5.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-over-slf4j-1.6.
> 6.jar:/usr/lib/apache/storm/1.0.2/lib/disruptor-3.3.2.jar:/
> usr/lib/apache/storm/1.0.2/lib/reflectasm-1.10.1.jar:/
> usr/lib/apache/storm/1.0.2/lib/asm-5.0.3.jar:/usr/lib/
> apache/storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
> org.apache.storm.command.list
> 12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while trying
> to get leader nimbus info from nimbus-stormcluster.gobalto.com. will
> retry with a different seed host.
> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
> java.net.ConnectException: Connection refused
> at org.apache.storm.security.auth.TBackoffConnect.
> retryNext(TBackoffConnect.java:64) ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
> [storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.command.list$_main.invoke(list.clj:22)
> [storm-core-1.0.2.jar:1.0.2]
> at clojure.lang.AFn.applyToHelper(AFn.java:152) [clojure-1.7.0.jar:?]
> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
> at org.apache.storm.command.list.main(Unknown Source)
> [storm-core-1.0.2.jar:1.0.2]
> Caused by: org.apache.storm.thrift.transport.TTransportException:
> java.net.ConnectException: Connection refused
> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.SimpleTransportPlugin.
> connect(SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
> ~[storm-core-1.0.2.jar:1.0.2]
> ... 8 more
> Caused by: java.net.ConnectException: Connection refused
> at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_91]
> at java.net.AbstractPlainSocketImpl.doConnect(
> AbstractPlainSocketImpl.java:350) ~[?:1.8.0_91]
> at java.net.AbstractPlainSocketImpl.connectToAddress(
> AbstractPlainSocketImpl.java:206) ~[?:1.8.0_91]
> at java.net.AbstractPlainSocketImpl.connect(
> AbstractPlainSocketImpl.java:188) ~[?:1.8.0_91]
> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
> ~[?:1.8.0_91]
> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.SimpleTransportPlugin.
> connect(SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
> ~[storm-core-1.0.2.jar:1.0.2]
> ... 8 more
> 22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while trying
> to get leader nimbus info from nimbus2-stormcluster.gobalto.com. will
> retry with a different seed host.
> java.lang.RuntimeException: org.apache.storm.thrift.transport.TTransportException:
> java.net.ConnectException: Connection refused
> at org.apache.storm.security.auth.TBackoffConnect.
> retryNext(TBackoffConnect.java:64) ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
> [storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.command.list$_main.invoke(list.clj:22)
> [storm-core-1.0.2.jar:1.0.2]
> at clojure.lang.AFn.applyToHelper(AFn.java:152) [clojure-1.7.0.jar:?]
> at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
> at org.apache.storm.command.list.main(Unknown Source)
> [storm-core-1.0.2.jar:1.0.2]
> Caused by: org.apache.storm.thrift.transport.TTransportException:
> java.net.ConnectException: Connection refused
> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.SimpleTransportPlugin.
> connect(SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
> ~[storm-core-1.0.2.jar:1.0.2]
> ... 8 more
> Caused by: java.net.ConnectException: Connection refused
> at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_91]
> at java.net.AbstractPlainSocketImpl.doConnect(
> AbstractPlainSocketImpl.java:350) ~[?:1.8.0_91]
> at java.net.AbstractPlainSocketImpl.connectToAddress(
> AbstractPlainSocketImpl.java:206) ~[?:1.8.0_91]
> at java.net.AbstractPlainSocketImpl.connect(
> AbstractPlainSocketImpl.java:188) ~[?:1.8.0_91]
> at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
> ~[?:1.8.0_91]
> at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
> at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
> ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.SimpleTransportPlugin.
> connect(SimpleTransportPlugin.java:103) ~[storm-core-1.0.2.jar:1.0.2]
> at org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
> ~[storm-core-1.0.2.jar:1.0.2]
> ... 8 more
> Exception in thread "main" org.apache.storm.utils.NimbusLeaderNotFoundException:
> Could not find leader nimbus from seed hosts ["
> nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did
> you specify a valid list of nimbus hosts for config nimbus.seeds?
> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(
> NimbusClient.java:90)
> at org.apache.storm.command.list$_main.invoke(list.clj:22)
> at clojure.lang.AFn.applyToHelper(AFn.java:152)
> at clojure.lang.AFn.applyTo(AFn.java:144)
> at org.apache.storm.command.list.main(Unknown Source)
>
>
> On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <jm...@gobalto.com>
> wrote:
>
>>
>> *Ideas?*
>> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
>> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
>> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
>> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
>> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
>> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
>> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
>> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
>> at clojure.core$some.invoke(core.clj:2570)
>> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
>> at clojure.lang.RestFn.applyTo(RestFn.java:139)
>> at clojure.core$apply.invoke(core.clj:632)
>> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
>> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
>> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
>> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
>> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
>> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
>> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
>> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
>> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
>> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
>> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
>> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
>> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
>> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
>> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
>> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:498)
>> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
>> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
>> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
>> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
>> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
>> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
>> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
>> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
>> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
>> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
>> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
>> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
>> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
>> at java.lang.Thread.run(Thread.java:745)
>>
>>
>>
>> --
>>
>> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>>
>
>
>
> --
>
> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>
Re: NimbusLeaderNotFoundException problem?
Posted by Joaquin Menchaca <jm...@gobalto.com>.
Also:
$ storm list
Running: /usr/lib/jvm/java-8-oracle/bin/java -client -Ddaemon.name=
-Dstorm.options= -Dstorm.home=/usr/lib/apache/storm/1.0.2
-Dstorm.log.dir=/usr/lib/apache/storm/1.0.2/logs
-Djava.library.path=/usr/local/lib:/opt/local/lib:/usr/lib
-Dstorm.conf.file= -cp
/usr/lib/apache/storm/1.0.2/lib/clojure-1.7.0.jar:/usr/lib/apache/storm/1.0.2/lib/minlog-1.3.0.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-api-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/storm-core-1.0.2.jar:/usr/lib/apache/storm/1.0.2/lib/storm-rename-hack-1.0.2.jar:/usr/lib/apache/storm/1.0.2/lib/slf4j-api-1.7.7.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-slf4j-impl-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/kryo-3.0.3.jar:/usr/lib/apache/storm/1.0.2/lib/objenesis-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-core-2.1.jar:/usr/lib/apache/storm/1.0.2/lib/servlet-api-2.5.jar:/usr/lib/apache/storm/1.0.2/lib/log4j-over-slf4j-1.6.6.jar:/usr/lib/apache/storm/1.0.2/lib/disruptor-3.3.2.jar:/usr/lib/apache/storm/1.0.2/lib/reflectasm-1.10.1.jar:/usr/lib/apache/storm/1.0.2/lib/asm-5.0.3.jar:/usr/lib/apache/storm/1.0.2/conf:/usr/lib/apache/storm/1.0.2/bin
org.apache.storm.command.list
12035 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while trying
to get leader nimbus info from nimbus-stormcluster.gobalto.com. will retry
with a different seed host.
java.lang.RuntimeException:
org.apache.storm.thrift.transport.TTransportException:
java.net.ConnectException: Connection refused
at
org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
~[storm-core-1.0.2.jar:1.0.2]
at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
[storm-core-1.0.2.jar:1.0.2]
at org.apache.storm.command.list$_main.invoke(list.clj:22)
[storm-core-1.0.2.jar:1.0.2]
at clojure.lang.AFn.applyToHelper(AFn.java:152) [clojure-1.7.0.jar:?]
at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
at org.apache.storm.command.list.main(Unknown Source)
[storm-core-1.0.2.jar:1.0.2]
Caused by: org.apache.storm.thrift.transport.TTransportException:
java.net.ConnectException: Connection refused
at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.SimpleTransportPlugin.connect(SimpleTransportPlugin.java:103)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
~[storm-core-1.0.2.jar:1.0.2]
... 8 more
Caused by: java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_91]
at
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
~[?:1.8.0_91]
at
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
~[?:1.8.0_91]
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
~[?:1.8.0_91]
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
~[?:1.8.0_91]
at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.SimpleTransportPlugin.connect(SimpleTransportPlugin.java:103)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
~[storm-core-1.0.2.jar:1.0.2]
... 8 more
22095 [main] WARN o.a.s.u.NimbusClient - Ignoring exception while trying
to get leader nimbus info from nimbus2-stormcluster.gobalto.com. will retry
with a different seed host.
java.lang.RuntimeException:
org.apache.storm.thrift.transport.TTransportException:
java.net.ConnectException: Connection refused
at
org.apache.storm.security.auth.TBackoffConnect.retryNext(TBackoffConnect.java:64)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:56)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.ThriftClient.reconnect(ThriftClient.java:99)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.ThriftClient.<init>(ThriftClient.java:69)
~[storm-core-1.0.2.jar:1.0.2]
at org.apache.storm.utils.NimbusClient.<init>(NimbusClient.java:106)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:66)
[storm-core-1.0.2.jar:1.0.2]
at org.apache.storm.command.list$_main.invoke(list.clj:22)
[storm-core-1.0.2.jar:1.0.2]
at clojure.lang.AFn.applyToHelper(AFn.java:152) [clojure-1.7.0.jar:?]
at clojure.lang.AFn.applyTo(AFn.java:144) [clojure-1.7.0.jar:?]
at org.apache.storm.command.list.main(Unknown Source)
[storm-core-1.0.2.jar:1.0.2]
Caused by: org.apache.storm.thrift.transport.TTransportException:
java.net.ConnectException: Connection refused
at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:226)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.SimpleTransportPlugin.connect(SimpleTransportPlugin.java:103)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
~[storm-core-1.0.2.jar:1.0.2]
... 8 more
Caused by: java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_91]
at
java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
~[?:1.8.0_91]
at
java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
~[?:1.8.0_91]
at
java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
~[?:1.8.0_91]
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
~[?:1.8.0_91]
at java.net.Socket.connect(Socket.java:589) ~[?:1.8.0_91]
at org.apache.storm.thrift.transport.TSocket.open(TSocket.java:221)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.thrift.transport.TFramedTransport.open(TFramedTransport.java:81)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.SimpleTransportPlugin.connect(SimpleTransportPlugin.java:103)
~[storm-core-1.0.2.jar:1.0.2]
at
org.apache.storm.security.auth.TBackoffConnect.doConnectWithRetry(TBackoffConnect.java:53)
~[storm-core-1.0.2.jar:1.0.2]
... 8 more
Exception in thread "main"
org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader
nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "
nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus
hosts for config nimbus.seeds?
at
org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
at org.apache.storm.command.list$_main.invoke(list.clj:22)
at clojure.lang.AFn.applyToHelper(AFn.java:152)
at clojure.lang.AFn.applyTo(AFn.java:144)
at org.apache.storm.command.list.main(Unknown Source)
On Mon, Dec 12, 2016 at 3:39 PM, Joaquin Menchaca <jm...@gobalto.com>
wrote:
>
> *Ideas?*
> org.apache.storm.utils.NimbusLeaderNotFoundException: Could not find leader nimbus from seed hosts ["nimbus-stormcluster.gobalto.com" "nimbus2-stormcluster.gobalto.com"]. Did you specify a valid list of nimbus hosts for config nimbus.seeds?
> at org.apache.storm.utils.NimbusClient.getConfiguredClientAs(NimbusClient.java:90)
> at org.apache.storm.ui.core$cluster_summary.invoke(core.clj:351)
> at org.apache.storm.ui.core$fn__12593.invoke(core.clj:931)
> at org.apache.storm.shade.compojure.core$make_route$fn__4631.invoke(core.clj:100)
> at org.apache.storm.shade.compojure.core$if_route$fn__4619.invoke(core.clj:46)
> at org.apache.storm.shade.compojure.core$if_method$fn__4612.invoke(core.clj:31)
> at org.apache.storm.shade.compojure.core$routing$fn__4637.invoke(core.clj:113)
> at clojure.core$some.invoke(core.clj:2570)
> at org.apache.storm.shade.compojure.core$routing.doInvoke(core.clj:113)
> at clojure.lang.RestFn.applyTo(RestFn.java:139)
> at clojure.core$apply.invoke(core.clj:632)
> at org.apache.storm.shade.compojure.core$routes$fn__4641.invoke(core.clj:118)
> at org.apache.storm.shade.ring.middleware.json$wrap_json_params$fn__12065.invoke(json.clj:56)
> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
> at org.apache.storm.shade.ring.middleware.reload$wrap_reload$fn__11217.invoke(reload.clj:22)
> at org.apache.storm.ui.helpers$requests_middleware$fn__6019.invoke(helpers.clj:50)
> at org.apache.storm.ui.core$catch_errors$fn__12786.invoke(core.clj:1225)
> at org.apache.storm.shade.ring.middleware.keyword_params$wrap_keyword_params$fn__5686.invoke(keyword_params.clj:35)
> at org.apache.storm.shade.ring.middleware.nested_params$wrap_nested_params$fn__5729.invoke(nested_params.clj:84)
> at org.apache.storm.shade.ring.middleware.params$wrap_params$fn__5658.invoke(params.clj:64)
> at org.apache.storm.shade.ring.middleware.multipart_params$wrap_multipart_params$fn__5766.invoke(multipart_params.clj:118)
> at org.apache.storm.shade.ring.middleware.flash$wrap_flash$fn__5981.invoke(flash.clj:35)
> at org.apache.storm.shade.ring.middleware.session$wrap_session$fn__5967.invoke(session.clj:98)
> at org.apache.storm.shade.ring.util.servlet$make_service_method$fn__5516.invoke(servlet.clj:127)
> at org.apache.storm.shade.ring.util.servlet$servlet$fn__5520.invoke(servlet.clj:136)
> at org.apache.storm.shade.ring.util.servlet.proxy$javax.servlet.http.HttpServlet$ff19274a.service(Unknown Source)
> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:654)
> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1320)
> at org.apache.storm.logging.filters.AccessLoggingFilter.handle(AccessLoggingFilter.java:47)
> at org.apache.storm.logging.filters.AccessLoggingFilter.doFilter(AccessLoggingFilter.java:39)
> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
> at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:93)
> at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28)
> at org.apache.storm.ui.helpers$x_frame_options_filter_handler$fn__6112.invoke(helpers.clj:189)
> at org.apache.storm.ui.helpers.proxy$java.lang.Object$Filter$abec9a8f.doFilter(Unknown Source)
> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.handle(CrossOriginFilter.java:247)
> at org.apache.storm.shade.org.eclipse.jetty.servlets.CrossOriginFilter.doFilter(CrossOriginFilter.java:210)
> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1291)
> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:443)
> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1044)
> at org.apache.storm.shade.org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:372)
> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:978)
> at org.apache.storm.shade.org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
> at org.apache.storm.shade.org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
> at org.apache.storm.shade.org.eclipse.jetty.server.Server.handle(Server.java:369)
> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:486)
> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:933)
> at org.apache.storm.shade.org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:995)
> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
> at org.apache.storm.shade.org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
> at org.apache.storm.shade.org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:668)
> at org.apache.storm.shade.org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
> at org.apache.storm.shade.org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
> at java.lang.Thread.run(Thread.java:745)
>
>
>
> --
>
> 是故勝兵先勝而後求戰，敗兵先戰而後求勝。
>
--
是故勝兵先勝而後求戰，敗兵先戰而後求勝。