You are viewing a plain text version of this content. The canonical link for it is here.

Posted to users@activemq.apache.org by kal123 <kp...@gmail.com> on 2013/11/05 18:33:52 UTC

activemq 5.9 Replicated LevelDB FlushCommand cannot cast to Message exception after failover

We received this error after the failover any idea why LevelDbClient is
getting these error, causes the broker to stop.

.activemq.leveldb.replicated.MasterLevelDBStore | hawtdispatch-DEFAULT-2
2013-11-05 12:09:57,386 | INFO  | Stopping BrokerService[largeamq] due to
exception, java.io.IOException: org.apache.activemq.command.FlushCommand
cannot be cast to org.apache.activemq.command.Message |
org.apache.activemq.util.DefaultIOExceptionHandler | LevelDB IOException
handler.
java.io.IOException: org.apache.activemq.command.FlushCommand cannot be cast
to org.apache.activemq.command.Message
        at
org.apache.activemq.util.IOExceptionSupport.create(IOExceptionSupport.java:39)
        at
org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:543)
        at
org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:974)
        at
org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1270)
        at
org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1194)
        at
org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:708)
        at
org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:741)
        at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:106)
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:258)
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
        at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
        at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1876)
        at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2086)
        at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1581)
        at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1803)
        at
org.apache.activemq.broker.region.Queue.addSubscription(Queue.java:464)
        at
org.apache.activemq.broker.region.AbstractRegion.addConsumer(AbstractRegion.java:314)
        at
org.apache.activemq.broker.region.RegionBroker.addConsumer(RegionBroker.java:401)
        at
org.apache.activemq.broker.jmx.ManagedRegionBroker.addConsumer(ManagedRegionBroker.java:230)
        at
org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:97)
        at
org.apache.activemq.advisory.AdvisoryBroker.addConsumer(AdvisoryBroker.java:102)
        at
org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:97)
        at
org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:97)

...
also looks like it cannot stop itself:
2013-11-05 12:09:58,216 | ERROR | Failed to page in more queue messages  |
org.apache.activemq.broker.region.Queue | ActiveMQ BrokerService[largeamq]
Task-60
java.lang.RuntimeException: java.lang.RuntimeException: java.io.IOException
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.reset(AbstractStoreCursor.java:111)
        at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.reset(StoreQueueCursor.java:157)
        at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1875)
        at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2086)
        at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1581)
        at org.apache.activemq.broker.region.Queue.wakeup(Queue.java:1803)
        at
org.apache.activemq.broker.region.Queue.removeSubscription(Queue.java:575)
        at
org.apache.activemq.broker.region.AbstractRegion.removeConsumer(AbstractRegion.java:368)
        at
org.apache.activemq.broker.region.RegionBroker.removeConsumer(RegionBroker.java:412)
        at
org.apache.activemq.broker.jmx.ManagedRegionBroker.removeConsumer(ManagedRegionBroker.java:248)
        at
org.apache.activemq.broker.BrokerFilter.removeConsumer(BrokerFilter.java:132)
        at
org.apache.activemq.advisory.AdvisoryBroker.removeConsumer(AdvisoryBroker.java:264)
        at
org.apache.activemq.broker.BrokerFilter.removeConsumer(BrokerFilter.java:132)
        at
org.apache.activemq.broker.BrokerFilter.removeConsumer(BrokerFilter.java:132)
        at
org.apache.activemq.broker.MutableBrokerFilter.removeConsumer(MutableBrokerFilter.java:137)
        at
org.apache.activemq.broker.TransportConnection.processRemoveConsumer(TransportConnection.java:619)
        at
org.apache.activemq.broker.TransportConnection.processRemoveSession(TransportConnection.java:658)
        at
org.apache.activemq.broker.TransportConnection.processRemoveConnection(TransportConnection.java:770)
        at
org.apache.activemq.broker.TransportConnection.doStop(TransportConnection.java:1107)
        at
org.apache.activemq.broker.TransportConnection$4.run(TransportConnection.java:1037)
        at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:724)
Caused by: java.lang.RuntimeException: java.io.IOException
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:261)
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.reset(AbstractStoreCursor.java:108)
        ... 22 more
Caused by: java.io.IOException
        at
org.apache.activemq.util.IOExceptionSupport.create(IOExceptionSupport.java:39)
        at
org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:543)
        at
org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:974)
        at
org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1270)
        at
org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1194)
        at
org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:708)
        at
org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:741)
        at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:106)
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:258)
        ... 23 more
Caused by: java.lang.NullPointerException
        at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply$mcV$sp(LevelDBClient.scala:1271)
        at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1271)
        at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$collectionCursor$1.apply(LevelDBClient.scala:1271)
        at
org.apache.activemq.leveldb.LevelDBClient.usingIndex(LevelDBClient.scala:968)
        at
org.apache.activemq.leveldb.LevelDBClient$$anonfun$might_fail_using_index$1.apply(LevelDBClient.scala:974)
        at
org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:540)
        ... 30 more
2013-11-05 12:10:03,219 | INFO  | The connection to
'tcp://10.44.175.235:43508' is taking a long time to shutdown. |
org.apache.activemq.broker.TransportConnection | IOExceptionHandler:
stopping BrokerService[largeamq]
2013-11-05 12:10:08,221 | INFO  | The connection to
'tcp://10.44.175.235:43508' is taking a long time to shutdown. |
org.apache.activemq.broker.TransportConnection | IOExceptionHandler:
stopping BrokerService[largeamq]





--
View this message in context: http://activemq.2283324.n4.nabble.com/activemq-5-9-Replicated-LevelDB-FlushCommand-cannot-cast-to-Message-exception-after-failover-tp4673914.html
Sent from the ActiveMQ - User mailing list archive at Nabble.com.

Re: activemq 5.9 Replicated LevelDB FlushCommand cannot cast to Message exception after failover

Posted by Timothy Bish <ta...@gmail.com>.

On 11/12/2013 12:06 PM, kal123 wrote:
> How I see following:
> Is leveldb putting msg (Eg. WireFormatInfo, FlushCOmmand?) in store that
> should not go in store?
>
>
> evelDBStore | hawtdispatch-DEFAULT-3
> 2013-11-12 11:40:50,486 | INFO  | Stopping BrokerService[largeamq] due to
> exception, java.io.IOException: org.apache.activemq.command.WireFormatInfo
> cannot be cast to org.apache.activemq.command.Message |
> org.apache.activemq.util.DefaultIOExceptionHandler | LevelDB IOException
> handler.
> java.io.IOException: org.apache.activemq.command.WireFormatInfo cannot be
> cast to org.apache.activemq.command.Message
>          at
> org.apache.activemq.util.IOExceptionSupport.create(IOExceptionSupport.java:39)
>          at
> org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:543)
>          at
> org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:974)
>          at
> org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1270)
>          at
> org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1194)
>          at
> org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:708)
>          at
> org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:741)
>          at
> org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:106)
>          at
> org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:258)
>          at
> org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
>          at
> org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
>          at
> org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1876)
>          at
> org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2086)
>          at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1581)
>          at
> org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:129)
>          at
> org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:47)
>          at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>          at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>
>
>
>
> --
> View this message in context: http://activemq.2283324.n4.nabble.com/activemq-5-9-Replicated-LevelDB-FlushCommand-cannot-cast-to-Message-exception-after-failover-tp4673914p4674231.html
> Sent from the ActiveMQ - User mailing list archive at Nabble.com.
>
There has been a lot of fixing in the LevelDB area of late, you are 
encouraged to move to a 5.10 SNAPSHOT build.

-- 
Tim Bish
Sr Software Engineer | RedHat Inc.
tim.bish@redhat.com | www.fusesource.com | www.redhat.com
skype: tabish121 | twitter: @tabish121
blog: http://timbish.blogspot.com/

Re: activemq 5.9 Replicated LevelDB FlushCommand cannot cast to Message exception after failover

Posted by kal123 <kp...@gmail.com>.

How I see following:
Is leveldb putting msg (Eg. WireFormatInfo, FlushCOmmand?) in store that
should not go in store?


evelDBStore | hawtdispatch-DEFAULT-3
2013-11-12 11:40:50,486 | INFO  | Stopping BrokerService[largeamq] due to
exception, java.io.IOException: org.apache.activemq.command.WireFormatInfo
cannot be cast to org.apache.activemq.command.Message |
org.apache.activemq.util.DefaultIOExceptionHandler | LevelDB IOException
handler.
java.io.IOException: org.apache.activemq.command.WireFormatInfo cannot be
cast to org.apache.activemq.command.Message
        at
org.apache.activemq.util.IOExceptionSupport.create(IOExceptionSupport.java:39)
        at
org.apache.activemq.leveldb.LevelDBClient.might_fail(LevelDBClient.scala:543)
        at
org.apache.activemq.leveldb.LevelDBClient.might_fail_using_index(LevelDBClient.scala:974)
        at
org.apache.activemq.leveldb.LevelDBClient.collectionCursor(LevelDBClient.scala:1270)
        at
org.apache.activemq.leveldb.LevelDBClient.queueCursor(LevelDBClient.scala:1194)
        at
org.apache.activemq.leveldb.DBManager.cursorMessages(DBManager.scala:708)
        at
org.apache.activemq.leveldb.LevelDBStore$LevelDBMessageStore.recoverNextMessages(LevelDBStore.scala:741)
        at
org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:106)
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:258)
        at
org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:145)
        at
org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:131)
        at
org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1876)
        at
org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2086)
        at org.apache.activemq.broker.region.Queue.iterate(Queue.java:1581)
        at
org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:129)
        at
org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:47)
        at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)




--
View this message in context: http://activemq.2283324.n4.nabble.com/activemq-5-9-Replicated-LevelDB-FlushCommand-cannot-cast-to-Message-exception-after-failover-tp4673914p4674231.html
Sent from the ActiveMQ - User mailing list archive at Nabble.com.