You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@storm.apache.org by "RAbreu (JIRA)" <ji...@apache.org> on 2017/09/06 18:56:00 UTC
[jira] [Commented] (STORM-2723) Nimbus crashes on joining cluster
[ https://issues.apache.org/jira/browse/STORM-2723?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16155879#comment-16155879 ]
RAbreu commented on STORM-2723:
-------------------------------
Shared storage is being used for storm/stormData
> Nimbus crashes on joining cluster
> ---------------------------------
>
> Key: STORM-2723
> URL: https://issues.apache.org/jira/browse/STORM-2723
> Project: Apache Storm
> Issue Type: Bug
> Components: blobstore
> Affects Versions: 1.1.0
> Reporter: RAbreu
>
> Cluster with N nodes and with running topologies. N new nodes join and the old machines start to be disconnected.
> Some of the new nimbus fail with this message:
> {code:java}
> 2017-09-06T16:30:53.551Z cluster [INFO] setup-path/blobstore/Topology-1-1504685635-stormconf.
> ser/node02:6627-1
> 2017-09-06T16:30:53.608Z nimbus [ERROR] Error when processing event
> java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: or
> g.apache.storm.thrift.transport.TTransportException
> at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:98) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.daemon.nimbus$fn__10607.invoke(nimbus.clj:1458) ~[storm-core-1.1.0.jar:1.1.0]
> at clojure.lang.MultiFn.invoke(MultiFn.java:233) ~[clojure-1.7.0.jar:?]
> at org.apache.storm.daemon.nimbus$fn__11005$exec_fn__1364__auto____11006$fn__11021.invoke(nimbus.clj:2460) ~[storm-core-1.1.0.j
> ar:1.1.0]
> at org.apache.storm.timer$schedule_recurring$this__1737.invoke(timer.clj:105) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.timer$mk_timer$fn__1720$fn__1721.invoke(timer.clj:50) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.timer$mk_timer$fn__1720.invoke(timer.clj:42) ~[storm-core-1.1.0.jar:1.1.0]
> at clojure.lang.AFn.run(AFn.java:22) ~[clojure-1.7.0.jar:?]
> at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
> Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.storm.th
> rift.transport.TTransportException
> at org.apache.storm.blobstore.BlobSynchronizer.updateKeySetForBlobStore(BlobSynchronizer.java:120) ~[storm-core-1.1.0.jar:1.1.0
> ]
> at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:77) ~[storm-core-1.1.0.jar:1.1.0]
> ... 8 more
> Caused by: java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException: org.apache.storm.thrift.transport.TTransportExc
> eption
> at org.apache.storm.blobstore.BlobStoreUtils.updateKeyForBlobStore(BlobStoreUtils.java:266) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.BlobSynchronizer.updateKeySetForBlobStore(BlobSynchronizer.java:117) ~[storm-core-1.1.0.jar:1.1.0
> ]
> at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:77) ~[storm-core-1.1.0.jar:1.1.0]
> ... 8 more
> Caused by: java.lang.RuntimeException: java.io.IOException: org.apache.storm.thrift.transport.TTransportException
> at org.apache.storm.blobstore.BlobStoreUtils.downloadUpdatedBlob(BlobStoreUtils.java:194) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.BlobStoreUtils.updateKeyForBlobStore(BlobStoreUtils.java:258) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.BlobSynchronizer.updateKeySetForBlobStore(BlobSynchronizer.java:117) ~[storm-core-1.1.0.jar:1.1.0
> ]
> at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:77) ~[storm-core-1.1.0.jar:1.1.0]
> ... 8 more
> Caused by: java.io.IOException: org.apache.storm.thrift.transport.TTransportException
> at org.apache.storm.blobstore.NimbusBlobStore$NimbusDownloadInputStream.read(NimbusBlobStore.java:156) ~[storm-core-1.1.0.jar:1
> .1.0]
> at org.apache.storm.blobstore.NimbusBlobStore$NimbusDownloadInputStream.read(NimbusBlobStore.java:182) ~[storm-core-1.1.0.jar:1
> .1.0]
> at org.apache.storm.blobstore.BlobStoreUtils.downloadUpdatedBlob(BlobStoreUtils.java:186) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.BlobStoreUtils.updateKeyForBlobStore(BlobStoreUtils.java:258) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.BlobSynchronizer.updateKeySetForBlobStore(BlobSynchronizer.java:117) ~[storm-core-1.1.0.jar:1.1.0
> ]
> at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:77) ~[storm-core-1.1.0.jar:1.1.0]
> ... 8 more
> Caused by: org.apache.storm.thrift.transport.TTransportException
> at org.apache.storm.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.transport.TTransport.readAll(TTransport.java:86) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.transport.TFramedTransport.readFrame(TFramedTransport.java:129) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.transport.TFramedTransport.read(TFramedTransport.java:101) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.transport.TTransport.readAll(TTransport.java:86) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.thrift.TServiceClient.receiveBase(TServiceClient.java:77) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.generated.Nimbus$Client.recv_downloadBlobChunk(Nimbus.java:866) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.generated.Nimbus$Client.downloadBlobChunk(Nimbus.java:853) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.NimbusBlobStore$NimbusDownloadInputStream.readMore(NimbusBlobStore.java:168) ~[storm-core-1.1.0.j
> ar:1.1.0]
> at org.apache.storm.blobstore.NimbusBlobStore$NimbusDownloadInputStream.read(NimbusBlobStore.java:146) ~[storm-core-1.1.0.jar:1
> .1.0]
> at org.apache.storm.blobstore.NimbusBlobStore$NimbusDownloadInputStream.read(NimbusBlobStore.java:182) ~[storm-core-1.1.0.jar:1
> .1.0]
> at org.apache.storm.blobstore.BlobStoreUtils.downloadUpdatedBlob(BlobStoreUtils.java:186) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.BlobStoreUtils.updateKeyForBlobStore(BlobStoreUtils.java:258) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.blobstore.BlobSynchronizer.updateKeySetForBlobStore(BlobSynchronizer.java:117) ~[storm-core-1.1.0.jar:1.1.0
> ]
> at org.apache.storm.blobstore.BlobSynchronizer.syncBlobs(BlobSynchronizer.java:77) ~[storm-core-1.1.0.jar:1.1.0]
> ... 8 more
> 2017-09-06T16:30:53.618Z util [ERROR] Halting process: ("Error when processing an event")
> java.lang.RuntimeException: ("Error when processing an event")
> at org.apache.storm.util$exit_process_BANG_.doInvoke(util.clj:341) ~[storm-core-1.1.0.jar:1.1.0]
> at clojure.lang.RestFn.invoke(RestFn.java:423) ~[clojure-1.7.0.jar:?]
> at org.apache.storm.daemon.nimbus$nimbus_data$fn__9808.invoke(nimbus.clj:212) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.timer$mk_timer$fn__1720$fn__1721.invoke(timer.clj:71) ~[storm-core-1.1.0.jar:1.1.0]
> at org.apache.storm.timer$mk_timer$fn__1720.invoke(timer.clj:42) ~[storm-core-1.1.0.jar:1.1.0]
> at clojure.lang.AFn.run(AFn.java:22) ~[clojure-1.7.0.jar:?]
> at java.lang.Thread.run(Thread.java:745) [?:1.8.0_60]
> 2017-09-06T16:30:53.619Z nimbus [INFO] Shutting down master
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)