You are viewing a plain text version of this content. The canonical link for it is here.

Posted to users@activemq.apache.org by "senthil.arumugam" <se...@gmail.com> on 2013/12/22 12:53:26 UTC

MQ broker crash

Hi,

We are using ActiveMQ 5.5.1 and recently the broker crashed. It failed to
restart until we cleared all the data files.

Any help to understand the rootcause or workaround to handle this issue ?


Below are the exceptions noticed in activemq.log,

2013-12-21 20:33:16,207 | ERROR | KahaDB failed to store to Journal |
org.apache.activemq.store.kahadb.MessageDatabase |
ConcurrentQueueStoreAndDispatch
java.io.EOFException: Chunk stream does not exist at page: 9449
        at
org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:454)
        at org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)
        at
org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:428)
        at org.apache.kahadb.page.Transaction.load(Transaction.java:404)
        at org.apache.kahadb.page.Transaction.load(Transaction.java:361)
        at org.apache.kahadb.index.BTreeIndex.loadNode(BTreeIndex.java:262)
        at org.apache.kahadb.index.BTreeNode.getChild(BTreeNode.java:225)
        at org.apache.kahadb.index.BTreeNode.getLeafNode(BTreeNode.java:672)
        at org.apache.kahadb.index.BTreeNode.put(BTreeNode.java:368)
        at org.apache.kahadb.index.BTreeIndex.put(BTreeIndex.java:189)
        at
org.apache.activemq.store.kahadb.MessageDatabase.upadateIndex(MessageDatabase.java:993)
        at
org.apache.activemq.store.kahadb.MessageDatabase$14.execute(MessageDatabase.java:872)
        at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
        at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:870)
        at
org.apache.activemq.store.kahadb.MessageDatabase$13.visit(MessageDatabase.java:828)
        at
org.apache.activemq.store.kahadb.data.KahaAddMessageCommand.visit(KahaAddMessageCommand.java:241)
        at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:825)
        at
org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:751)
        at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.addMessage(KahaDBStore.java:366)
        at
org.apache.activemq.store.kahadb.KahaDBStore$StoreQueueTask.run(KahaDBStore.java:1123)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
        at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
        at java.lang.Thread.run(Thread.java:619)


2013-12-21 20:33:15,897 | WARN  | Async error occurred:
java.lang.ClassCastException: java.lang.Long cannot be cast to
java.lang.String | org.apache.activemq.broker.TransportConnection.Service |
ActiveMQ Transport: tcp:///10.128.24.198:5
4154
java.lang.ClassCastException: java.lang.Long cannot be cast to
java.lang.String
        at
org.apache.kahadb.util.StringMarshaller.writePayload(StringMarshaller.java:28)
        at
org.apache.kahadb.index.BTreeNode$Marshaller.writePayload(BTreeNode.java:163)
        at
org.apache.kahadb.index.BTreeNode$Marshaller.writePayload(BTreeNode.java:147)
        at org.apache.kahadb.page.Transaction.store(Transaction.java:249)
        at org.apache.kahadb.index.BTreeIndex.storeNode(BTreeIndex.java:289)
        at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:316)
        at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:287)
        at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:287)
        at org.apache.kahadb.index.BTreeIndex.remove(BTreeIndex.java:194)
        at
org.apache.activemq.store.kahadb.MessageDatabase.updateIndex(MessageDatabase.java:1029)
        at
org.apache.activemq.store.kahadb.MessageDatabase$15.execute(MessageDatabase.java:890)
        at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
        at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:888)
        at
org.apache.activemq.store.kahadb.MessageDatabase$13.visit(MessageDatabase.java:833)
        at
org.apache.activemq.store.kahadb.data.KahaRemoveMessageCommand.visit(KahaRemoveMessageCommand.java:220)
        at
org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:825)
        at
org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:751)
        at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.removeMessage(KahaDBStore.java:378)
        at
org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.removeAsyncMessage(KahaDBStore.java:350)
        at
org.apache.activemq.store.kahadb.KahaDBTransactionStore.removeAsyncMessage(KahaDBTransactionStore.java:457)
        at
org.apache.activemq.store.kahadb.KahaDBTransactionStore$1.removeAsyncMessage(KahaDBTransactionStore.java:171)
        at
org.apache.activemq.broker.region.Queue.acknowledge(Queue.java:800)
        at
org.apache.activemq.broker.region.Queue.removeMessage(Queue.java:1530)
        at
org.apache.activemq.broker.region.QueueSubscription.acknowledge(QueueSubscription.java:59)
        at
org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:253)
        at
org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
        at
org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:546)
        at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:78)
        at
org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:78)
        at
org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
        at
org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
        at
org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:468)
        at org.apache.activemq.command.MessageAck.visit(MessageAck.java:214)
        at
org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:310)
        at
org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:184)
        at
org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
        at
org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
        at
org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
        at
org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
        at
org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:222)
        at
org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:204)
        at java.lang.Thread.run(Thread.java:619)




--
View this message in context: http://activemq.2283324.n4.nabble.com/MQ-broker-crash-tp4675798.html
Sent from the ActiveMQ - User mailing list archive at Nabble.com.

Re: MQ broker crash

Posted by Christian Posta <ch...@gmail.com>.

Did you have any corruption of kahadb files outside of the broker?

On Sun, Dec 22, 2013 at 4:53 AM, senthil.arumugam
<se...@gmail.com> wrote:
> Hi,
>
> We are using ActiveMQ 5.5.1 and recently the broker crashed. It failed to
> restart until we cleared all the data files.
>
> Any help to understand the rootcause or workaround to handle this issue ?
>
>
> Below are the exceptions noticed in activemq.log,
>
> 2013-12-21 20:33:16,207 | ERROR | KahaDB failed to store to Journal |
> org.apache.activemq.store.kahadb.MessageDatabase |
> ConcurrentQueueStoreAndDispatch
> java.io.EOFException: Chunk stream does not exist at page: 9449
>         at
> org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:454)
>         at org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)
>         at
> org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:428)
>         at org.apache.kahadb.page.Transaction.load(Transaction.java:404)
>         at org.apache.kahadb.page.Transaction.load(Transaction.java:361)
>         at org.apache.kahadb.index.BTreeIndex.loadNode(BTreeIndex.java:262)
>         at org.apache.kahadb.index.BTreeNode.getChild(BTreeNode.java:225)
>         at org.apache.kahadb.index.BTreeNode.getLeafNode(BTreeNode.java:672)
>         at org.apache.kahadb.index.BTreeNode.put(BTreeNode.java:368)
>         at org.apache.kahadb.index.BTreeIndex.put(BTreeIndex.java:189)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.upadateIndex(MessageDatabase.java:993)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase$14.execute(MessageDatabase.java:872)
>         at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:870)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase$13.visit(MessageDatabase.java:828)
>         at
> org.apache.activemq.store.kahadb.data.KahaAddMessageCommand.visit(KahaAddMessageCommand.java:241)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:825)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:751)
>         at
> org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.addMessage(KahaDBStore.java:366)
>         at
> org.apache.activemq.store.kahadb.KahaDBStore$StoreQueueTask.run(KahaDBStore.java:1123)
>         at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
>         at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
>         at java.lang.Thread.run(Thread.java:619)
>
>
> 2013-12-21 20:33:15,897 | WARN  | Async error occurred:
> java.lang.ClassCastException: java.lang.Long cannot be cast to
> java.lang.String | org.apache.activemq.broker.TransportConnection.Service |
> ActiveMQ Transport: tcp:///10.128.24.198:5
> 4154
> java.lang.ClassCastException: java.lang.Long cannot be cast to
> java.lang.String
>         at
> org.apache.kahadb.util.StringMarshaller.writePayload(StringMarshaller.java:28)
>         at
> org.apache.kahadb.index.BTreeNode$Marshaller.writePayload(BTreeNode.java:163)
>         at
> org.apache.kahadb.index.BTreeNode$Marshaller.writePayload(BTreeNode.java:147)
>         at org.apache.kahadb.page.Transaction.store(Transaction.java:249)
>         at org.apache.kahadb.index.BTreeIndex.storeNode(BTreeIndex.java:289)
>         at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:316)
>         at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:287)
>         at org.apache.kahadb.index.BTreeNode.remove(BTreeNode.java:287)
>         at org.apache.kahadb.index.BTreeIndex.remove(BTreeIndex.java:194)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.updateIndex(MessageDatabase.java:1029)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase$15.execute(MessageDatabase.java:890)
>         at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:888)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase$13.visit(MessageDatabase.java:833)
>         at
> org.apache.activemq.store.kahadb.data.KahaRemoveMessageCommand.visit(KahaRemoveMessageCommand.java:220)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.process(MessageDatabase.java:825)
>         at
> org.apache.activemq.store.kahadb.MessageDatabase.store(MessageDatabase.java:751)
>         at
> org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.removeMessage(KahaDBStore.java:378)
>         at
> org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.removeAsyncMessage(KahaDBStore.java:350)
>         at
> org.apache.activemq.store.kahadb.KahaDBTransactionStore.removeAsyncMessage(KahaDBTransactionStore.java:457)
>         at
> org.apache.activemq.store.kahadb.KahaDBTransactionStore$1.removeAsyncMessage(KahaDBTransactionStore.java:171)
>         at
> org.apache.activemq.broker.region.Queue.acknowledge(Queue.java:800)
>         at
> org.apache.activemq.broker.region.Queue.removeMessage(Queue.java:1530)
>         at
> org.apache.activemq.broker.region.QueueSubscription.acknowledge(QueueSubscription.java:59)
>         at
> org.apache.activemq.broker.region.PrefetchSubscription.acknowledge(PrefetchSubscription.java:253)
>         at
> org.apache.activemq.broker.region.AbstractRegion.acknowledge(AbstractRegion.java:383)
>         at
> org.apache.activemq.broker.region.RegionBroker.acknowledge(RegionBroker.java:546)
>         at
> org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:78)
>         at
> org.apache.activemq.broker.BrokerFilter.acknowledge(BrokerFilter.java:78)
>         at
> org.apache.activemq.broker.TransactionBroker.acknowledge(TransactionBroker.java:197)
>         at
> org.apache.activemq.broker.MutableBrokerFilter.acknowledge(MutableBrokerFilter.java:87)
>         at
> org.apache.activemq.broker.TransportConnection.processMessageAck(TransportConnection.java:468)
>         at org.apache.activemq.command.MessageAck.visit(MessageAck.java:214)
>         at
> org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:310)
>         at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:184)
>         at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
>         at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
>         at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
>         at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
>         at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:222)
>         at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:204)
>         at java.lang.Thread.run(Thread.java:619)
>
>
>
>
> --
> View this message in context: http://activemq.2283324.n4.nabble.com/MQ-broker-crash-tp4675798.html
> Sent from the ActiveMQ - User mailing list archive at Nabble.com.



-- 
Christian Posta
http://www.christianposta.com/blog
twitter: @christianposta