You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@activemq.apache.org by Aravind Chennuru <Ar...@netboss.com> on 2012/11/06 17:49:20 UTC
ActiveMQ locking all the threads [5.5.1]
Hi All,
I am facing a problem on some of our production servers with ActiveMQ locking the threads and the messages are no more sent to the consumers. Did someone notice similar problem and know if there is a way to address this?
It is not a deadlock but when this happens the messages are stopped being sent.
"ActiveMQ Transport: tcp:///192.168.18.129:57610" daemon prio=3 tid=0x00000001027aa000 nid=0x3a waiting for monitor entry [0xfffffffee6efe000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:423)
- waiting to lock <0xffffffff005397c0> (a org.apache.activemq.broker.region.Topic)
at org.apache.activemq.broker.region.Topic.send(Topic.java:404)
at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
at org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
- locked <0xffffffff00767bb8> (a org.apache.activemq.transport.InactivityMonitor$1)
at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
at java.lang.Thread.run(Thread.java:662)
"ActiveMQ Transport: tcp:///192.168.18.130:40617" daemon prio=3 tid=0x0000000102f53000 nid=0x46 runnable [0xfffffffee60fe000]
java.lang.Thread.State: RUNNABLE
at java.io.RandomAccessFile.readBytes(Native Method)
at java.io.RandomAccessFile.read(RandomAccessFile.java:338)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:397)
at java.io.RandomAccessFile.readFully(RandomAccessFile.java:377)
at org.apache.kahadb.page.PageFile.readPage(PageFile.java:779)
at org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:440)
at org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)
at org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:428)
at org.apache.kahadb.page.Transaction.load(Transaction.java:404)
at org.apache.kahadb.page.Transaction.load(Transaction.java:361)
at org.apache.activemq.store.kahadb.plist.PList.loadEntry(PList.java:468)
at org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:483)
at org.apache.activemq.store.kahadb.plist.PList$10.execute(PList.java:312)
at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
at org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:310)
- locked <0xffffffff004de4c0> (a java.lang.Object)
- locked <0xffffffff09a97068> (a org.apache.activemq.store.kahadb.plist.PList)
at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:500)
- locked <0xffffffff09a97068> (a org.apache.activemq.store.kahadb.plist.PList)
at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:473)
at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor.next(FilePendingMessageCursor.java:293)
- locked <0xffffffff013199f0> (a org.apache.activemq.broker.region.cursors.FilePendingMessageCursor)
at org.apache.activemq.broker.region.TopicSubscription.removeExpiredMessages(TopicSubscription.java:212)
at org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscription.java:149)
- locked <0xffffffff01316480> (a java.lang.Object)
at org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch(SimpleDispatchPolicy.java:48)
at org.apache.activemq.broker.region.Topic.dispatch(Topic.java:634)
at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:468)
- locked <0xffffffff005397c0> (a org.apache.activemq.broker.region.Topic)
at org.apache.activemq.broker.region.Topic.send(Topic.java:404)
at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
at org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
at org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
- locked <0xffffffff004ca288> (a org.apache.activemq.transport.InactivityMonitor$1)
at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
at java.lang.Thread.run(Thread.java:662)
Thanks & Regards,
Aravind Chennuru
RE: ActiveMQ locking all the threads [5.5.1]
Posted by Aravind Chennuru <Ar...@netboss.com>.
-rw-r--r-- 1 root root 33555136 Nov 6 16:24 db-2.log
-rw-r--r-- 1 root root 33554663 Nov 6 16:24 db-3.log
-rw-r--r-- 1 root root 33557047 Nov 6 16:24 db-6.log
-rw-r--r-- 1 root root 33556316 Nov 6 16:25 db-7.log
-rw-r--r-- 1 root root 33556009 Nov 6 16:25 db-8.log
-rw-r--r-- 1 root root 33555539 Nov 6 17:20 db-9.log
-rw-r--r-- 1 root root 33555787 Nov 6 19:16 db-10.log
-rw-r--r-- 1 root root 33030144 Nov 6 20:00 db-11.log
-rw-r--r-- 1 root root 80110072 Nov 6 20:01 tmpDB.redo
-rw-r--r-- 1 root root 398520320 Nov 6 20:01 tmpDB.data
We run broker as root and looking at the permissions this process should be able to read this data without issues, there is nothing in the logs to mention that it's failing to read these files.
-----Original Message-----
From: Christian Posta [mailto:christian.posta@gmail.com]
Sent: Tuesday, November 06, 2012 7:49 PM
To: users@activemq.apache.org
Subject: Re: ActiveMQ locking all the threads [5.5.1]
Looks like the subscription trying to read from the FilePendingMessageCursor's temp store is trying to read the contents of the store file, but is blocking on readFully()... keeping other threads from sending.
Any reason why it couldn't read the temp store?
On Tue, Nov 6, 2012 at 11:44 AM, Aravind Chennuru < Aravind.Chennuru@netboss.com> wrote:
> I attached the email about the way we use ActiveMQ, including the
> configuration file we used to start ActiveMQ.****
>
> ** **
>
> ** **
>
> ** **
>
> -----Original Message-----
> From: Gaurav Sharma [mailto:gaurav.cs.sharma@gmail.com]
> Sent: Tuesday, November 06, 2012 12:31 PM
> To: users@activemq.apache.org
> Subject: Re: ActiveMQ locking all the threads [5.5.1]
>
> ** **
>
> Mind sharing the messaging topology and how the broker is getting used
> by the app?****
>
> ** **
>
> On Nov 6, 2012, at 8:49, Aravind Chennuru
> <Ar...@netboss.com>
> wrote:****
>
> ** **
>
> > Hi All,****
>
> > ****
>
> > I am facing a problem on some of our production servers with
> > ActiveMQ
> locking the threads and the messages are no more sent to the
> consumers. Did someone notice similar problem and know if there is a way to address this?
> ****
>
> > It is not a deadlock but when this happens the messages are stopped
> being sent.****
>
> > ****
>
> > ****
>
> > "ActiveMQ Transport: tcp:///192.168.18.129:57610" daemon prio=3
> tid=0x00000001027aa000 nid=0x3a waiting for monitor entry
> [0xfffffffee6efe000]****
>
> > java.lang.Thread.State: BLOCKED (on object monitor)****
>
> > at
> org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:423)*
> ***
>
> > - waiting to lock <0xffffffff005397c0> (a
> org.apache.activemq.broker.region.Topic)****
>
> > at
> org.apache.activemq.broker.region.Topic.send(Topic.java:404)****
>
> > at
> org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.j
> ava:365)
> ****
>
> > at
> org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:
> 523)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)***
> *
>
> > at
> org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDe
> stinationBroker.java:96)
> ****
>
> > at
> org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.ja
> va:227)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)***
> *
>
> > at
> org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilte
> r.java:135)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.processMessage(Transpor
> tConnection.java:458)
> ****
>
> > at
> org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java
> :681)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.service(TransportConnec
> tion.java:306)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportCo
> nnection.java:179)
> ****
>
> > at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilte
> r.java:69)
> ****
>
> > at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireForma
> tNegotiator.java:113)
> ****
>
> > at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMo
> nitor.java:227)
> ****
>
> > - locked <0xffffffff00767bb8> (a
> org.apache.activemq.transport.InactivityMonitor$1)****
>
> > at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupp
> ort.java:83)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java
> :220)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:2
> 02)*
> ***
>
> > at java.lang.Thread.run(Thread.java:662)****
>
> > ****
>
> > "ActiveMQ Transport: tcp:///192.168.18.130:40617" daemon prio=3
> tid=0x0000000102f53000 nid=0x46 runnable [0xfffffffee60fe000]****
>
> > java.lang.Thread.State: RUNNABLE****
>
> > at java.io.RandomAccessFile.readBytes(Native
> > Method)****
>
> > at
> java.io.RandomAccessFile.read(RandomAccessFile.java:338)****
>
> > at
> java.io.RandomAccessFile.readFully(RandomAccessFile.java:397)****
>
> > at
> java.io.RandomAccessFile.readFully(RandomAccessFile.java:377)****
>
> > at
> org.apache.kahadb.page.PageFile.readPage(PageFile.java:779)****
>
> > at
> org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:440)***
> *
>
> > at
> org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)****
>
> > at
> org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:42
> 8)**
> **
>
> > at
> org.apache.kahadb.page.Transaction.load(Transaction.java:404)****
>
> > at
> org.apache.kahadb.page.Transaction.load(Transaction.java:361)****
>
> > at
> org.apache.activemq.store.kahadb.plist.PList.loadEntry(PList.java:468)
> ****
>
> > at
> org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:483)**
> **
>
> > at
> org.apache.activemq.store.kahadb.plist.PList$10.execute(PList.java:312
> )***
> *
>
> > at
> org.apache.kahadb.page.Transaction.execute(Transaction.java:728)****
>
> > at
> org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:310)**
> **
>
> > - locked <0xffffffff004de4c0> (a
> > java.lang.Object)****
>
> > - locked <0xffffffff09a97068> (a
> org.apache.activemq.store.kahadb.plist.PList)****
>
> > at
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$Dis
> kIterator.next(FilePendingMessageCursor.java:500)
> ****
>
> > - locked <0xffffffff09a97068> (a
> org.apache.activemq.store.kahadb.plist.PList)****
>
> > at
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$Dis
> kIterator.next(FilePendingMessageCursor.java:473)
> ****
>
> > at
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor.nex
> t(FilePendingMessageCursor.java:293)
> ****
>
> > - locked <0xffffffff013199f0> (a
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor)***
> *
>
> > at
> org.apache.activemq.broker.region.TopicSubscription.removeExpiredMessa
> ges(TopicSubscription.java:212)
> ****
>
> > at
> org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscript
> ion.java:149)
> ****
>
> > - locked <0xffffffff01316480> (a
> > java.lang.Object)****
>
> > at
> org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch
> (SimpleDispatchPolicy.java:48)
> ****
>
> > at
> org.apache.activemq.broker.region.Topic.dispatch(Topic.java:634)****
>
> > at
> org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:468)*
> ***
>
> > - locked <0xffffffff005397c0> (a
> org.apache.activemq.broker.region.Topic)****
>
> > at
> org.apache.activemq.broker.region.Topic.send(Topic.java:404)****
>
> > at
> org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.j
> ava:365)
> ****
>
> > at
> org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:
> 523)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)***
> *
>
> > at
> org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDe
> stinationBroker.java:96)
> ****
>
> > at
> org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.ja
> va:227)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)***
> *
>
> > at
> org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilte
> r.java:135)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.processMessage(Transpor
> tConnection.java:458)
> ****
>
> > at
> org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java
> :681)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.service(TransportConnec
> tion.java:306)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportCo
> nnection.java:179)
> ****
>
> > at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilte
> r.java:69)
> ****
>
> > at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireForma
> tNegotiator.java:113)
> ****
>
> > at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMo
> nitor.java:227)
> ****
>
> > - locked <0xffffffff004ca288> (a
> org.apache.activemq.transport.InactivityMonitor$1)****
>
> > at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupp
> ort.java:83)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java
> :220)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:2
> 02)*
> ***
>
> > at java.lang.Thread.run(Thread.java:662)****
>
> > ****
>
> > Thanks & Regards,****
>
> > Aravind Chennuru****
>
> ** **
>
--
*Christian Posta*
http://www.christianposta.com/blog
twitter: @christianposta
Re: ActiveMQ locking all the threads [5.5.1]
Posted by Christian Posta <ch...@gmail.com>.
Looks like the subscription trying to read from the
FilePendingMessageCursor's temp store is trying to read the contents of the
store file, but is blocking on readFully()... keeping other threads from
sending.
Any reason why it couldn't read the temp store?
On Tue, Nov 6, 2012 at 11:44 AM, Aravind Chennuru <
Aravind.Chennuru@netboss.com> wrote:
> I attached the email about the way we use ActiveMQ, including the
> configuration file we used to start ActiveMQ.****
>
> ** **
>
> ** **
>
> ** **
>
> -----Original Message-----
> From: Gaurav Sharma [mailto:gaurav.cs.sharma@gmail.com]
> Sent: Tuesday, November 06, 2012 12:31 PM
> To: users@activemq.apache.org
> Subject: Re: ActiveMQ locking all the threads [5.5.1]
>
> ** **
>
> Mind sharing the messaging topology and how the broker is getting used by
> the app?****
>
> ** **
>
> On Nov 6, 2012, at 8:49, Aravind Chennuru <Ar...@netboss.com>
> wrote:****
>
> ** **
>
> > Hi All,****
>
> > ****
>
> > I am facing a problem on some of our production servers with ActiveMQ
> locking the threads and the messages are no more sent to the consumers. Did
> someone notice similar problem and know if there is a way to address this?
> ****
>
> > It is not a deadlock but when this happens the messages are stopped
> being sent.****
>
> > ****
>
> > ****
>
> > "ActiveMQ Transport: tcp:///192.168.18.129:57610" daemon prio=3
> tid=0x00000001027aa000 nid=0x3a waiting for monitor entry
> [0xfffffffee6efe000]****
>
> > java.lang.Thread.State: BLOCKED (on object monitor)****
>
> > at
> org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:423)****
>
> > - waiting to lock <0xffffffff005397c0> (a
> org.apache.activemq.broker.region.Topic)****
>
> > at
> org.apache.activemq.broker.region.Topic.send(Topic.java:404)****
>
> > at
> org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
> ****
>
> > at
> org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)****
>
> > at
> org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
> ****
>
> > at
> org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)****
>
> > at
> org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
> ****
>
> > at
> org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
> ****
>
> > at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> ****
>
> > at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> ****
>
> > at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
> ****
>
> > - locked <0xffffffff00767bb8> (a
> org.apache.activemq.transport.InactivityMonitor$1)****
>
> > at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)*
> ***
>
> > at java.lang.Thread.run(Thread.java:662)****
>
> > ****
>
> > "ActiveMQ Transport: tcp:///192.168.18.130:40617" daemon prio=3
> tid=0x0000000102f53000 nid=0x46 runnable [0xfffffffee60fe000]****
>
> > java.lang.Thread.State: RUNNABLE****
>
> > at java.io.RandomAccessFile.readBytes(Native Method)****
>
> > at
> java.io.RandomAccessFile.read(RandomAccessFile.java:338)****
>
> > at
> java.io.RandomAccessFile.readFully(RandomAccessFile.java:397)****
>
> > at
> java.io.RandomAccessFile.readFully(RandomAccessFile.java:377)****
>
> > at
> org.apache.kahadb.page.PageFile.readPage(PageFile.java:779)****
>
> > at
> org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:440)****
>
> > at
> org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)****
>
> > at
> org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:428)**
> **
>
> > at
> org.apache.kahadb.page.Transaction.load(Transaction.java:404)****
>
> > at
> org.apache.kahadb.page.Transaction.load(Transaction.java:361)****
>
> > at
> org.apache.activemq.store.kahadb.plist.PList.loadEntry(PList.java:468)****
>
> > at
> org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:483)****
>
> > at
> org.apache.activemq.store.kahadb.plist.PList$10.execute(PList.java:312)***
> *
>
> > at
> org.apache.kahadb.page.Transaction.execute(Transaction.java:728)****
>
> > at
> org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:310)****
>
> > - locked <0xffffffff004de4c0> (a java.lang.Object)****
>
> > - locked <0xffffffff09a97068> (a
> org.apache.activemq.store.kahadb.plist.PList)****
>
> > at
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:500)
> ****
>
> > - locked <0xffffffff09a97068> (a
> org.apache.activemq.store.kahadb.plist.PList)****
>
> > at
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:473)
> ****
>
> > at
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor.next(FilePendingMessageCursor.java:293)
> ****
>
> > - locked <0xffffffff013199f0> (a
> org.apache.activemq.broker.region.cursors.FilePendingMessageCursor)****
>
> > at
> org.apache.activemq.broker.region.TopicSubscription.removeExpiredMessages(TopicSubscription.java:212)
> ****
>
> > at
> org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscription.java:149)
> ****
>
> > - locked <0xffffffff01316480> (a java.lang.Object)****
>
> > at
> org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch(SimpleDispatchPolicy.java:48)
> ****
>
> > at
> org.apache.activemq.broker.region.Topic.dispatch(Topic.java:634)****
>
> > at
> org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:468)****
>
> > - locked <0xffffffff005397c0> (a
> org.apache.activemq.broker.region.Topic)****
>
> > at
> org.apache.activemq.broker.region.Topic.send(Topic.java:404)****
>
> > at
> org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
> ****
>
> > at
> org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)****
>
> > at
> org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
> ****
>
> > at
> org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
> ****
>
> > at
> org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)****
>
> > at
> org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
> ****
>
> > at
> org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
> ****
>
> > at
> org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
> ****
>
> > at
> org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> ****
>
> > at
> org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> ****
>
> > at
> org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
> ****
>
> > - locked <0xffffffff004ca288> (a
> org.apache.activemq.transport.InactivityMonitor$1)****
>
> > at
> org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> ****
>
> > at
> org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)*
> ***
>
> > at java.lang.Thread.run(Thread.java:662)****
>
> > ****
>
> > Thanks & Regards,****
>
> > Aravind Chennuru****
>
> ** **
>
--
*Christian Posta*
http://www.christianposta.com/blog
twitter: @christianposta
RE: ActiveMQ locking all the threads [5.5.1]
Posted by Aravind Chennuru <Ar...@netboss.com>.
I attached the email about the way we use ActiveMQ, including the configuration file we used to start ActiveMQ.
-----Original Message-----
From: Gaurav Sharma [mailto:gaurav.cs.sharma@gmail.com]
Sent: Tuesday, November 06, 2012 12:31 PM
To: users@activemq.apache.org
Subject: Re: ActiveMQ locking all the threads [5.5.1]
Mind sharing the messaging topology and how the broker is getting used by the app?
On Nov 6, 2012, at 8:49, Aravind Chennuru <Ar...@netboss.com>> wrote:
> Hi All,
>
> I am facing a problem on some of our production servers with ActiveMQ locking the threads and the messages are no more sent to the consumers. Did someone notice similar problem and know if there is a way to address this?
> It is not a deadlock but when this happens the messages are stopped being sent.
>
>
> "ActiveMQ Transport: tcp:///192.168.18.129:57610" daemon prio=3 tid=0x00000001027aa000 nid=0x3a waiting for monitor entry [0xfffffffee6efe000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:423)
> - waiting to lock <0xffffffff005397c0> (a org.apache.activemq.broker.region.Topic)
> at org.apache.activemq.broker.region.Topic.send(Topic.java:404)
> at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
> at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
> at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
> at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
> at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
> at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
> at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
> at org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> at org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
> - locked <0xffffffff00767bb8> (a org.apache.activemq.transport.InactivityMonitor$1)
> at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
> at java.lang.Thread.run(Thread.java:662)
>
> "ActiveMQ Transport: tcp:///192.168.18.130:40617" daemon prio=3 tid=0x0000000102f53000 nid=0x46 runnable [0xfffffffee60fe000]
> java.lang.Thread.State: RUNNABLE
> at java.io.RandomAccessFile.readBytes(Native Method)
> at java.io.RandomAccessFile.read(RandomAccessFile.java:338)
> at java.io.RandomAccessFile.readFully(RandomAccessFile.java:397)
> at java.io.RandomAccessFile.readFully(RandomAccessFile.java:377)
> at org.apache.kahadb.page.PageFile.readPage(PageFile.java:779)
> at org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:440)
> at org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)
> at org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:428)
> at org.apache.kahadb.page.Transaction.load(Transaction.java:404)
> at org.apache.kahadb.page.Transaction.load(Transaction.java:361)
> at org.apache.activemq.store.kahadb.plist.PList.loadEntry(PList.java:468)
> at org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:483)
> at org.apache.activemq.store.kahadb.plist.PList$10.execute(PList.java:312)
> at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
> at org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:310)
> - locked <0xffffffff004de4c0> (a java.lang.Object)
> - locked <0xffffffff09a97068> (a org.apache.activemq.store.kahadb.plist.PList)
> at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:500)
> - locked <0xffffffff09a97068> (a org.apache.activemq.store.kahadb.plist.PList)
> at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:473)
> at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor.next(FilePendingMessageCursor.java:293)
> - locked <0xffffffff013199f0> (a org.apache.activemq.broker.region.cursors.FilePendingMessageCursor)
> at org.apache.activemq.broker.region.TopicSubscription.removeExpiredMessages(TopicSubscription.java:212)
> at org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscription.java:149)
> - locked <0xffffffff01316480> (a java.lang.Object)
> at org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch(SimpleDispatchPolicy.java:48)
> at org.apache.activemq.broker.region.Topic.dispatch(Topic.java:634)
> at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:468)
> - locked <0xffffffff005397c0> (a org.apache.activemq.broker.region.Topic)
> at org.apache.activemq.broker.region.Topic.send(Topic.java:404)
> at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
> at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
> at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
> at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
> at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
> at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
> at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
> at org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> at org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
> - locked <0xffffffff004ca288> (a org.apache.activemq.transport.InactivityMonitor$1)
> at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
> at java.lang.Thread.run(Thread.java:662)
>
> Thanks & Regards,
> Aravind Chennuru
Re: ActiveMQ locking all the threads [5.5.1]
Posted by Gaurav Sharma <ga...@gmail.com>.
Mind sharing the messaging topology and how the broker is getting used by the app?
On Nov 6, 2012, at 8:49, Aravind Chennuru <Ar...@netboss.com> wrote:
> Hi All,
>
> I am facing a problem on some of our production servers with ActiveMQ locking the threads and the messages are no more sent to the consumers. Did someone notice similar problem and know if there is a way to address this?
> It is not a deadlock but when this happens the messages are stopped being sent.
>
>
> "ActiveMQ Transport: tcp:///192.168.18.129:57610" daemon prio=3 tid=0x00000001027aa000 nid=0x3a waiting for monitor entry [0xfffffffee6efe000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:423)
> - waiting to lock <0xffffffff005397c0> (a org.apache.activemq.broker.region.Topic)
> at org.apache.activemq.broker.region.Topic.send(Topic.java:404)
> at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
> at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
> at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
> at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
> at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
> at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
> at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
> at org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> at org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
> - locked <0xffffffff00767bb8> (a org.apache.activemq.transport.InactivityMonitor$1)
> at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
> at java.lang.Thread.run(Thread.java:662)
>
> "ActiveMQ Transport: tcp:///192.168.18.130:40617" daemon prio=3 tid=0x0000000102f53000 nid=0x46 runnable [0xfffffffee60fe000]
> java.lang.Thread.State: RUNNABLE
> at java.io.RandomAccessFile.readBytes(Native Method)
> at java.io.RandomAccessFile.read(RandomAccessFile.java:338)
> at java.io.RandomAccessFile.readFully(RandomAccessFile.java:397)
> at java.io.RandomAccessFile.readFully(RandomAccessFile.java:377)
> at org.apache.kahadb.page.PageFile.readPage(PageFile.java:779)
> at org.apache.kahadb.page.Transaction$2.readPage(Transaction.java:440)
> at org.apache.kahadb.page.Transaction$2.<init>(Transaction.java:431)
> at org.apache.kahadb.page.Transaction.openInputStream(Transaction.java:428)
> at org.apache.kahadb.page.Transaction.load(Transaction.java:404)
> at org.apache.kahadb.page.Transaction.load(Transaction.java:361)
> at org.apache.activemq.store.kahadb.plist.PList.loadEntry(PList.java:468)
> at org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:483)
> at org.apache.activemq.store.kahadb.plist.PList$10.execute(PList.java:312)
> at org.apache.kahadb.page.Transaction.execute(Transaction.java:728)
> at org.apache.activemq.store.kahadb.plist.PList.getNext(PList.java:310)
> - locked <0xffffffff004de4c0> (a java.lang.Object)
> - locked <0xffffffff09a97068> (a org.apache.activemq.store.kahadb.plist.PList)
> at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:500)
> - locked <0xffffffff09a97068> (a org.apache.activemq.store.kahadb.plist.PList)
> at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor$DiskIterator.next(FilePendingMessageCursor.java:473)
> at org.apache.activemq.broker.region.cursors.FilePendingMessageCursor.next(FilePendingMessageCursor.java:293)
> - locked <0xffffffff013199f0> (a org.apache.activemq.broker.region.cursors.FilePendingMessageCursor)
> at org.apache.activemq.broker.region.TopicSubscription.removeExpiredMessages(TopicSubscription.java:212)
> at org.apache.activemq.broker.region.TopicSubscription.add(TopicSubscription.java:149)
> - locked <0xffffffff01316480> (a java.lang.Object)
> at org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch(SimpleDispatchPolicy.java:48)
> at org.apache.activemq.broker.region.Topic.dispatch(Topic.java:634)
> at org.apache.activemq.broker.region.Topic.doMessageSend(Topic.java:468)
> - locked <0xffffffff005397c0> (a org.apache.activemq.broker.region.Topic)
> at org.apache.activemq.broker.region.Topic.send(Topic.java:404)
> at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:365)
> at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:523)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
> at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:227)
> at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
> at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
> at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:458)
> at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:681)
> at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:306)
> at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:179)
> at org.apache.activemq.transport.TransportFilter.onCommand(TransportFilter.java:69)
> at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
> at org.apache.activemq.transport.InactivityMonitor.onCommand(InactivityMonitor.java:227)
> - locked <0xffffffff004ca288> (a org.apache.activemq.transport.InactivityMonitor$1)
> at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
> at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:220)
> at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:202)
> at java.lang.Thread.run(Thread.java:662)
>
> Thanks & Regards,
> Aravind Chennuru