You are viewing a plain text version of this content. The canonical link for it is here.
Posted to users@activemq.apache.org by chris <em...@christianhoffmann.info> on 2011/03/14 17:39:06 UTC
ActiveMQ-CPP: deadlock when broker restarts and failover is used
Hi all,
we seem to have a deadlock situation when a broker shutdown happens.
We're in the middle of onMessage callback and trying to call commit.
The ActiveMQConsumer::dispatch method holds the "unconsumedMessages"
lock and waits for the "reconnectMutex" when committing. The failover
transport calls clearMessagesInProgress which tries to get hold of the
"unconsumedMessages" lock but holds the "reconnectMutex" in
FailoverTransport::handleTransportFailure. The two stack traces below
should show what happens.
We're using ActiveMQ-CPP 3.2.5 with a 5.3.2 broker.
Kind regards,
Chris
One thread here:
ntdll.dll!77dbf8c1()
[Frames below may be incorrect and/or missing, no symbols loaded
for
ntdll.dll]
ntdll.dll!77dbf8c1()
ntdll.dll!77dd8dd4()
kernel32.dll!775c14dd()
>
activemq-cppd.dll!decaf::internal::util::concurrent::MutexImpl::lock(decaf::util::concurrent::MutexHandle
* handle=0x039e0ed0) Line 68 + 0xa bytes C++
activemq-cppd.dll!decaf::util::concurrent::Mutex::lock() Line 75 +
0xe bytes C++
activemq-cppd.dll!decaf::util::StlQueue<decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
>::lock() Line 254 C++
activemq-cppd.dll!activemq::core::MessageDispatchChannel::lock()
Line 154 C++
activemq-cppd.dll!decaf::util::concurrent::Lock::lock() Line 55
C++
activemq-cppd.dll!decaf::util::concurrent::Lock::Lock(decaf::util::concurrent::Synchronizable
* object=0x039d4c48, const bool intiallyLocked=true) Line 34 C++
activemq-cppd.dll!activemq::core::ActiveMQConsumer::clearMessagesInProgress()
Line 1112 + 0x17 bytes C++
activemq-cppd.dll!activemq::core::ActiveMQSession::clearMessagesInProgress()
Line 240 C++
activemq-cppd.dll!activemq::core::ActiveMQConnection::transportInterrupted()
Line 705 C++
activemq-cppd.dll!activemq::transport::TransportFilter::transportInterrupted()
Line 67 + 0x13 bytes C++
activemq-cppd.dll!activemq::transport::failover::FailoverTransport::handleTransportFailure(const
decaf::lang::Exception & error={...}) Line 479 C++
activemq-cppd.dll!activemq::transport::failover::FailoverTransportListener::onException(const
decaf::lang::Exception & ex={...}) Line 97 + 0xf bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
decaf::lang::Exception & ex={...}) Line 42 C++
activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
decaf::lang::Exception & ex={...}) Line 42 C++
activemq-cppd.dll!activemq::transport::inactivity::InactivityMonitor::onException(const
decaf::lang::Exception & ex={...}) Line 276 C++
activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
decaf::lang::Exception & ex={...}) Line 42 C++
activemq-cppd.dll!activemq::transport::IOTransport::fire(decaf::lang::Exception
& ex={...}) Line 73 + 0x17 bytes C++
activemq-cppd.dll!activemq::transport::IOTransport::run() Line 247
C++
activemq-cppd.dll!decaf::lang::ThreadProperties::runCallback(decaf::lang::ThreadProperties
* properties=0x039cb968) Line 135 + 0x11 bytes C++
activemq-cppd.dll!`anonymous namespace'::threadWorker(void *
arg=0x039cb968) Line 208 + 0x9 bytes C++
msvcr100d.dll!_callthreadstartex() Line 314 + 0xf bytes C
msvcr100d.dll!_threadstartex(void * ptd=0x0396e030) Line 297 C
kernel32.dll!775c33ca()
ntdll.dll!77dd9ed2()
ntdll.dll!77dd9ea5()
Dispatch thread here:
ntdll.dll!77dbf8c1()
[Frames below may be incorrect and/or missing, no symbols loaded
for
ntdll.dll]
ntdll.dll!77dbf8c1()
ntdll.dll!77dd8dd4()
ntdll.dll!77dd8cb8()
>
activemq-cppd.dll!decaf::internal::util::concurrent::MutexImpl::lock(decaf::util::concurrent::MutexHandle
* handle=0x039b6988) Line 68 + 0xa bytes C++
activemq-cppd.dll!decaf::util::concurrent::Mutex::lock() Line 75 +
0xe bytes C++
activemq-cppd.dll!decaf::util::concurrent::Lock::lock() Line 55
C++
activemq-cppd.dll!decaf::util::concurrent::Lock::Lock(decaf::util::concurrent::Synchronizable
* object=0x039d4974, const bool intiallyLocked=true) Line 34 C++
activemq-cppd.dll!activemq::transport::failover::FailoverTransport::oneway(const
decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
& command={...}) Line 186 + 0x20 bytes C++
activemq-cppd.dll!activemq::transport::correlator::ResponseCorrelator::oneway(const
decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
& command={...}) Line 86 C++
activemq-cppd.dll!activemq::core::ActiveMQConnection::oneway(decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
command={...}) Line 742 C++
activemq-cppd.dll!activemq::core::ActiveMQSession::oneway(decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
command={...}) Line 904 C++
activemq-cppd.dll!activemq::core::ActiveMQConsumer::acknowledge()
Line 861 C++
activemq-cppd.dll!activemq::core::TransactionSynhcronization::beforeEnd()
Line 86 C++
activemq-cppd.dll!activemq::core::ActiveMQTransactionContext::beforeEnd()
Line 192 + 0x41 bytes C++
activemq-cppd.dll!activemq::core::ActiveMQTransactionContext::commit()
Line 130 C++
activemq-cppd.dll!activemq::core::ActiveMQSession::commit() Line
190 C++
<<<<<onMessage call removed>>>>>>>
activemq-cppd.dll!activemq::core::ActiveMQConsumer::dispatch(const
decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
& dispatch={...}) Line 1021 C++
activemq-cppd.dll!activemq::core::ActiveMQSessionExecutor::dispatch(const
decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
& dispatch={...}) Line 129 + 0x15 bytes C++
activemq-cppd.dll!activemq::core::ActiveMQSessionExecutor::iterate()
Line 167 C++
activemq-cppd.dll!activemq::threads::DedicatedTaskRunner::run()
Line 111 + 0x19 bytes C++
activemq-cppd.dll!decaf::lang::ThreadProperties::runCallback(decaf::lang::ThreadProperties
* properties=0x039cbd58) Line 135 + 0x11 bytes C++
activemq-cppd.dll!`anonymous namespace'::threadWorker(void *
arg=0x039cbd58) Line 208 + 0x9 bytes C++
msvcr100d.dll!_callthreadstartex() Line 314 + 0xf bytes C
msvcr100d.dll!_threadstartex(void * ptd=0x0396dba0) Line 297 C
kernel32.dll!775c33ca()
ntdll.dll!77dd9ed2()
ntdll.dll!77dd9ea5()
Re: ActiveMQ-CPP: deadlock when broker restarts and failover is used
Posted by Oscar Pernas <os...@pernas.es>.
This bug was reported some time ago. If you want to make a workaround, if
you are consuming messages in asynchronous (by callback), change to
synchronous consuming mode. In my case its resolved it.
2011/3/14 chris <em...@christianhoffmann.info>
> Hi all,
>
> we seem to have a deadlock situation when a broker shutdown happens.
> We're in the middle of onMessage callback and trying to call commit.
> The ActiveMQConsumer::dispatch method holds the "unconsumedMessages"
> lock and waits for the "reconnectMutex" when committing. The failover
> transport calls clearMessagesInProgress which tries to get hold of the
> "unconsumedMessages" lock but holds the "reconnectMutex" in
> FailoverTransport::handleTransportFailure. The two stack traces below
> should show what happens.
>
> We're using ActiveMQ-CPP 3.2.5 with a 5.3.2 broker.
>
> Kind regards,
> Chris
>
> One thread here:
> ntdll.dll!77dbf8c1()
> [Frames below may be incorrect and/or missing, no symbols loaded
> for
> ntdll.dll]
> ntdll.dll!77dbf8c1()
> ntdll.dll!77dd8dd4()
> kernel32.dll!775c14dd()
> >
>
> activemq-cppd.dll!decaf::internal::util::concurrent::MutexImpl::lock(decaf::util::concurrent::MutexHandle
> * handle=0x039e0ed0) Line 68 + 0xa bytes C++
> activemq-cppd.dll!decaf::util::concurrent::Mutex::lock() Line 75 +
> 0xe bytes C++
>
>
> activemq-cppd.dll!decaf::util::StlQueue<decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
> >::lock() Line 254 C++
> activemq-cppd.dll!activemq::core::MessageDispatchChannel::lock()
> Line 154 C++
> activemq-cppd.dll!decaf::util::concurrent::Lock::lock() Line 55
> C++
>
>
> activemq-cppd.dll!decaf::util::concurrent::Lock::Lock(decaf::util::concurrent::Synchronizable
> * object=0x039d4c48, const bool intiallyLocked=true) Line 34 C++
>
>
> activemq-cppd.dll!activemq::core::ActiveMQConsumer::clearMessagesInProgress()
> Line 1112 + 0x17 bytes C++
>
>
> activemq-cppd.dll!activemq::core::ActiveMQSession::clearMessagesInProgress()
> Line 240 C++
>
>
> activemq-cppd.dll!activemq::core::ActiveMQConnection::transportInterrupted()
> Line 705 C++
>
>
> activemq-cppd.dll!activemq::transport::TransportFilter::transportInterrupted()
> Line 67 + 0x13 bytes C++
>
>
> activemq-cppd.dll!activemq::transport::failover::FailoverTransport::handleTransportFailure(const
> decaf::lang::Exception & error={...}) Line 479 C++
>
>
> activemq-cppd.dll!activemq::transport::failover::FailoverTransportListener::onException(const
> decaf::lang::Exception & ex={...}) Line 97 + 0xf bytes C++
> activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
> decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
>
> activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
> decaf::lang::Exception & ex={...}) Line 42 C++
> activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
> decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
>
> activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
> decaf::lang::Exception & ex={...}) Line 42 C++
>
>
> activemq-cppd.dll!activemq::transport::inactivity::InactivityMonitor::onException(const
> decaf::lang::Exception & ex={...}) Line 276 C++
> activemq-cppd.dll!activemq::transport::TransportFilter::fire(const
> decaf::lang::Exception & ex={...}) Line 49 + 0x17 bytes C++
>
> activemq-cppd.dll!activemq::transport::TransportFilter::onException(const
> decaf::lang::Exception & ex={...}) Line 42 C++
>
>
> activemq-cppd.dll!activemq::transport::IOTransport::fire(decaf::lang::Exception
> & ex={...}) Line 73 + 0x17 bytes C++
> activemq-cppd.dll!activemq::transport::IOTransport::run() Line 247
> C++
>
>
> activemq-cppd.dll!decaf::lang::ThreadProperties::runCallback(decaf::lang::ThreadProperties
> * properties=0x039cb968) Line 135 + 0x11 bytes C++
> activemq-cppd.dll!`anonymous namespace'::threadWorker(void *
> arg=0x039cb968) Line 208 + 0x9 bytes C++
> msvcr100d.dll!_callthreadstartex() Line 314 + 0xf bytes C
> msvcr100d.dll!_threadstartex(void * ptd=0x0396e030) Line 297 C
> kernel32.dll!775c33ca()
> ntdll.dll!77dd9ed2()
> ntdll.dll!77dd9ea5()
>
> Dispatch thread here:
>
> ntdll.dll!77dbf8c1()
> [Frames below may be incorrect and/or missing, no symbols loaded
> for
> ntdll.dll]
> ntdll.dll!77dbf8c1()
> ntdll.dll!77dd8dd4()
> ntdll.dll!77dd8cb8()
> >
>
> activemq-cppd.dll!decaf::internal::util::concurrent::MutexImpl::lock(decaf::util::concurrent::MutexHandle
> * handle=0x039b6988) Line 68 + 0xa bytes C++
> activemq-cppd.dll!decaf::util::concurrent::Mutex::lock() Line 75 +
> 0xe bytes C++
> activemq-cppd.dll!decaf::util::concurrent::Lock::lock() Line 55
> C++
>
>
> activemq-cppd.dll!decaf::util::concurrent::Lock::Lock(decaf::util::concurrent::Synchronizable
> * object=0x039d4974, const bool intiallyLocked=true) Line 34 C++
>
>
> activemq-cppd.dll!activemq::transport::failover::FailoverTransport::oneway(const
>
> decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
> & command={...}) Line 186 + 0x20 bytes C++
>
>
> activemq-cppd.dll!activemq::transport::correlator::ResponseCorrelator::oneway(const
>
> decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
> & command={...}) Line 86 C++
>
>
> activemq-cppd.dll!activemq::core::ActiveMQConnection::oneway(decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
> command={...}) Line 742 C++
>
>
> activemq-cppd.dll!activemq::core::ActiveMQSession::oneway(decaf::lang::Pointer<activemq::commands::Command,decaf::util::concurrent::atomic::AtomicRefCounter>
> command={...}) Line 904 C++
> activemq-cppd.dll!activemq::core::ActiveMQConsumer::acknowledge()
> Line 861 C++
>
> activemq-cppd.dll!activemq::core::TransactionSynhcronization::beforeEnd()
> Line 86 C++
>
> activemq-cppd.dll!activemq::core::ActiveMQTransactionContext::beforeEnd()
> Line 192 + 0x41 bytes C++
>
> activemq-cppd.dll!activemq::core::ActiveMQTransactionContext::commit()
> Line 130 C++
> activemq-cppd.dll!activemq::core::ActiveMQSession::commit() Line
> 190 C++
> <<<<<onMessage call removed>>>>>>>
> activemq-cppd.dll!activemq::core::ActiveMQConsumer::dispatch(const
>
> decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
> & dispatch={...}) Line 1021 C++
>
> activemq-cppd.dll!activemq::core::ActiveMQSessionExecutor::dispatch(const
>
> decaf::lang::Pointer<activemq::commands::MessageDispatch,decaf::util::concurrent::atomic::AtomicRefCounter>
> & dispatch={...}) Line 129 + 0x15 bytes C++
>
> activemq-cppd.dll!activemq::core::ActiveMQSessionExecutor::iterate()
> Line 167 C++
> activemq-cppd.dll!activemq::threads::DedicatedTaskRunner::run()
> Line 111 + 0x19 bytes C++
>
>
> activemq-cppd.dll!decaf::lang::ThreadProperties::runCallback(decaf::lang::ThreadProperties
> * properties=0x039cbd58) Line 135 + 0x11 bytes C++
> activemq-cppd.dll!`anonymous namespace'::threadWorker(void *
> arg=0x039cbd58) Line 208 + 0x9 bytes C++
> msvcr100d.dll!_callthreadstartex() Line 314 + 0xf bytes C
> msvcr100d.dll!_threadstartex(void * ptd=0x0396dba0) Line 297 C
> kernel32.dll!775c33ca()
> ntdll.dll!77dd9ed2()
> ntdll.dll!77dd9ea5()
>
--
Óscar Pernas Plaza.