You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by "Dmitry Mitko (JIRA)" <ji...@apache.org> on 2014/07/02 22:40:27 UTC
[jira] [Comment Edited] (CASSANDRA-6565) New node refuses to join
the ring.
[ https://issues.apache.org/jira/browse/CASSANDRA-6565?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14050682#comment-14050682 ]
Dmitry Mitko edited comment on CASSANDRA-6565 at 7/2/14 8:40 PM:
-----------------------------------------------------------------
The same issue with 2.0.7. We can't add more nodes into our cluster anymore.
{noformat}
WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:02:41,966 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error
java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:04,954 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error
java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,563 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error
java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
ERROR [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,565 StreamSession.java (line 420) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Streaming error occurred
java.io.IOException: Too many retries for Header (cfId: 9d1ff8a6-e358-302c-8b7b-49ca458ae31b, #23, version: jb, estimated keys: 5888, transfer size: 26198229, compressed?: true)
at org.apache.cassandra.streaming.StreamSession.doRetry(StreamSession.java:544)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:51)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
... 4 more
{noformat}
and then
{noformat}
WARN [NonPeriodicTasks:1] 2014-07-02 21:46:14,451 StreamResultFuture.java (line 215) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Stream failed
ERROR [main] 2014-07-02 21:46:14,452 CassandraDaemon.java (line 497) Exception encountered during startup
java.lang.RuntimeException: Error during boostrap: Stream failed
at org.apache.cassandra.dht.BootStrapper.bootstrap(BootStrapper.java:86)
at org.apache.cassandra.service.StorageService.bootstrap(StorageService.java:1035)
at org.apache.cassandra.service.StorageService.joinTokenRing(StorageService.java:797)
at org.apache.cassandra.service.StorageService.initServer(StorageService.java:615)
at org.apache.cassandra.service.StorageService.initServer(StorageService.java:505)
at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:362)
at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:480)
at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:569)
Caused by: org.apache.cassandra.streaming.StreamException: Stream failed
at org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85)
at com.google.common.util.concurrent.Futures$4.run(Futures.java:1160)
at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)
at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)
at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202)
at org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:216)
at org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:191)
at org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:331)
at org.apache.cassandra.streaming.StreamSession.maybeCompleted(StreamSession.java:615)
at org.apache.cassandra.streaming.StreamSession.taskCompleted(StreamSession.java:566)
at org.apache.cassandra.streaming.StreamReceiveTask$OnCompletionRunnable.run(StreamReceiveTask.java:120)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
{noformat}
and the end is
{noformat}
ERROR [Thread-31] 2014-07-02 21:46:16,479 CassandraDaemon.java (line 198) Exception in thread Thread[Thread-31,5,main]
java.util.concurrent.RejectedExecutionException: ThreadPoolExecutor has shut down
at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor$1.rejectedExecution(DebuggableThreadPoolExecutor.java:61)
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor.execute(DebuggableThreadPoolExecutor.java:136)
at org.apache.cassandra.net.MessagingService.receive(MessagingService.java:709)
at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:161)
at org.apache.cassandra.net.IncomingTcpConnection.handleModernVersion(IncomingTcpConnection.java:130)
at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:74)
{noformat}
Please, investigate this issue or help with some workaround to force the joining process somehow. Maybe, just ignore failed streams will be good idea? We are using replication factor = 3.
was (Author: dv1983):
The same issue. We can't add more nodes into our cluster anymore.
{noformat}
WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:02:41,966 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error
java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:04,954 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error
java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
WARN [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,563 StreamSession.java (line 540) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Retrying for following error
java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
ERROR [STREAM-IN-/192.168.114.102] 2014-07-02 18:03:59,565 StreamSession.java (line 420) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Streaming error occurred
java.io.IOException: Too many retries for Header (cfId: 9d1ff8a6-e358-302c-8b7b-49ca458ae31b, #23, version: jb, estimated keys: 5888, transfer size: 26198229, compressed?: true)
at org.apache.cassandra.streaming.StreamSession.doRetry(StreamSession.java:544)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:51)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:37)
at org.apache.cassandra.streaming.messages.StreamMessage.deserialize(StreamMessage.java:55)
at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:283)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.AssertionError
at org.apache.cassandra.streaming.compress.CompressedInputStream.read(CompressedInputStream.java:94)
at java.io.InputStream.read(InputStream.java:170)
at java.io.InputStream.skip(InputStream.java:222)
at org.apache.cassandra.streaming.StreamReader.drain(StreamReader.java:117)
at org.apache.cassandra.streaming.compress.CompressedStreamReader.read(CompressedStreamReader.java:89)
at org.apache.cassandra.streaming.messages.IncomingFileMessage$1.deserialize(IncomingFileMessage.java:47)
... 4 more
{noformat}
and then
{noformat}
WARN [NonPeriodicTasks:1] 2014-07-02 21:46:14,451 StreamResultFuture.java (line 215) [Stream #730f6560-01fd-11e4-bcb9-3b37e5507e63] Stream failed
ERROR [main] 2014-07-02 21:46:14,452 CassandraDaemon.java (line 497) Exception encountered during startup
java.lang.RuntimeException: Error during boostrap: Stream failed
at org.apache.cassandra.dht.BootStrapper.bootstrap(BootStrapper.java:86)
at org.apache.cassandra.service.StorageService.bootstrap(StorageService.java:1035)
at org.apache.cassandra.service.StorageService.joinTokenRing(StorageService.java:797)
at org.apache.cassandra.service.StorageService.initServer(StorageService.java:615)
at org.apache.cassandra.service.StorageService.initServer(StorageService.java:505)
at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:362)
at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:480)
at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:569)
Caused by: org.apache.cassandra.streaming.StreamException: Stream failed
at org.apache.cassandra.streaming.management.StreamEventJMXNotifier.onFailure(StreamEventJMXNotifier.java:85)
at com.google.common.util.concurrent.Futures$4.run(Futures.java:1160)
at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)
at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)
at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202)
at org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:216)
at org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:191)
at org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:331)
at org.apache.cassandra.streaming.StreamSession.maybeCompleted(StreamSession.java:615)
at org.apache.cassandra.streaming.StreamSession.taskCompleted(StreamSession.java:566)
at org.apache.cassandra.streaming.StreamReceiveTask$OnCompletionRunnable.run(StreamReceiveTask.java:120)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
{noformat}
and the end is
{noformat}
ERROR [Thread-31] 2014-07-02 21:46:16,479 CassandraDaemon.java (line 198) Exception in thread Thread[Thread-31,5,main]
java.util.concurrent.RejectedExecutionException: ThreadPoolExecutor has shut down
at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor$1.rejectedExecution(DebuggableThreadPoolExecutor.java:61)
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
at org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor.execute(DebuggableThreadPoolExecutor.java:136)
at org.apache.cassandra.net.MessagingService.receive(MessagingService.java:709)
at org.apache.cassandra.net.IncomingTcpConnection.receiveMessage(IncomingTcpConnection.java:161)
at org.apache.cassandra.net.IncomingTcpConnection.handleModernVersion(IncomingTcpConnection.java:130)
at org.apache.cassandra.net.IncomingTcpConnection.run(IncomingTcpConnection.java:74)
{noformat}
Please, investigate this issue or help with some workaround to force the joining process somehow. Maybe, just ignore failed streams will be good idea? We are using replication factor = 3.
> New node refuses to join the ring.
> ----------------------------------
>
> Key: CASSANDRA-6565
> URL: https://issues.apache.org/jira/browse/CASSANDRA-6565
> Project: Cassandra
> Issue Type: Bug
> Reporter: Shao-Chuan Wang
>
> We have 30 nodes in one DC, 25 nodes in another. We are running 2.0.1.
> Two nodes are joining the ring, but one of them failed
> ARN [STREAM-IN-/10.4.197.53] 2014-01-09 19:41:40,418 StreamResultFuture.java (line 209) [Stream #e515d6e0-795d-11e3-b74a-b72892248056] Stream failed
> ERROR [main] 2014-01-09 19:41:40,418 CassandraDaemon.java (line 459) Exception encountered during startup
> java.lang.RuntimeException: Error during boostrap: Stream failed
> at org.apache.cassandra.dht.BootStrapper.bootstrap(BootStrapper.java:86)
> at org.apache.cassandra.service.StorageService.bootstrap(StorageService.java:901)
> at org.apache.cassandra.service.StorageService.joinTokenRing(StorageService.java:670)
> at org.apache.cassandra.service.StorageService.initServer(StorageService.java:529)
> at org.apache.cassandra.service.StorageService.initServer(StorageService.java:428)
> at org.apache.cassandra.service.CassandraDaemon.setup(CassandraDaemon.java:343)
> at org.apache.cassandra.service.CassandraDaemon.activate(CassandraDaemon.java:442)
> at org.apache.cassandra.service.CassandraDaemon.main(CassandraDaemon.java:485)
> Caused by: org.apache.cassandra.streaming.StreamException: Stream failed
> at org.apache.cassandra.streaming.StreamResultFuture.maybeComplete(StreamResultFuture.java:210)
> at org.apache.cassandra.streaming.StreamResultFuture.handleSessionComplete(StreamResultFuture.java:185)
> at org.apache.cassandra.streaming.StreamSession.closeSession(StreamSession.java:321)
> at org.apache.cassandra.streaming.StreamSession.complete(StreamSession.java:501) at org.apache.cassandra.streaming.StreamSession.messageReceived(Stre
> amSession.java:376) at org.apache.cassandra.streaming.ConnectionHandler$IncomingMessageHandler.run(ConnectionHandler.java:293)
> at java.lang.Thread.run(Thread.java:744)
> ERROR [StorageServiceShutdownHook] 2014-01-09 19:41:40,428 CassandraDaemon.java (line 185) Exception in thread Thread[StorageServiceShutdownHook,5,main]
> java.lang.NullPointerException
> at org.apache.cassandra.service.StorageService.stopRPCServer(StorageService.java:312)
> at org.apache.cassandra.service.StorageService.shutdownClientServers(StorageService.java:361)
> at org.apache.cassandra.service.StorageService.access$000(StorageService.java:96)
> at org.apache.cassandra.service.StorageService$1.runMayThrow(StorageService.java:494)
> at org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
> at java.lang.Thread.run(Thread.java:744)
--
This message was sent by Atlassian JIRA
(v6.2#6252)