You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@activemq.apache.org by "bogdan t. (Jira)" <ji...@apache.org> on 2023/10/18 10:53:00 UTC
[jira] [Updated] (AMQ-9342) deadlock between threads calling failover and rollback
[ https://issues.apache.org/jira/browse/AMQ-9342?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
bogdan t. updated AMQ-9342:
---------------------------
Description:
We encounter deadlock after failover - looks like order of locking in failover handling is different than locking order during rollback and thread are clashing:
Rollback:
- FifoMessageDispatchChannel
- then ActiveMQConnection
Failover handling:
- ActiveMQConnection
- then FifoMessageDispatchChannel
See the full stack traces:
{noformat}
"thread-doing-rollback":
at org.apache.activemq.ActiveMQConnection.getScheduler(ActiveMQConnection.java:2387)
- waiting to lock <0x00007f315a19d550> (a org.apache.activemq.ActiveMQConnection)
at org.apache.activemq.ActiveMQSession.getScheduler(ActiveMQSession.java:2254)
at org.apache.activemq.ActiveMQMessageConsumer.rollback(ActiveMQMessageConsumer.java:1356)
- locked <0x00007f37630c0d90> (a java.util.LinkedList)
- locked <0x00007f37630a4690> (a java.lang.Object)
at org.apache.activemq.ActiveMQMessageConsumer$5.afterRollback(ActiveMQMessageConsumer.java:1113)
at org.apache.activemq.TransactionContext.afterRollback(TransactionContext.java:166)
at org.apache.activemq.TransactionContext.commit(TransactionContext.java:344)
at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:583)
at org.apache.activemq.jms.pool.PooledSession.commit(PooledSession.java:167)
at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:218)
at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:776)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:680)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:331)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:270)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1237)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1227)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1120)
at java.lang.Thread.run(java.base@11.0.12/Thread.java:829)
"ActiveMQ Connection Executor: failover:(nio://....)&maxReconnectAttempts=0":
at org.apache.activemq.FifoMessageDispatchChannel.close(FifoMessageDispatchChannel.java:133)
- waiting to lock <0x00007f37630a4690> (a java.lang.Object)
at org.apache.activemq.ActiveMQMessageConsumer.dispose(ActiveMQMessageConsumer.java:872)
at org.apache.activemq.ActiveMQSession.dispose(ActiveMQSession.java:730)
- locked <0x00007f376311fd78> (a org.apache.activemq.ActiveMQSession)
at org.apache.activemq.ActiveMQConnection.close(ActiveMQConnection.java:671)
- locked <0x00007f315a19d550> (a org.apache.activemq.ActiveMQConnection)
at org.apache.activemq.jms.pool.ConnectionPool.close(ConnectionPool.java:176)
- locked <0x00007f315a230e50> (a org.apache.activemq.pool.PooledConnectionFactory$1)
at org.apache.activemq.jms.pool.ConnectionPool.onException(ConnectionPool.java:381)
at org.apache.activemq.ActiveMQConnection$5.run(ActiveMQConnection.java:1967)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.12/ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.12/ThreadPoolExecutor.java:628)
at java.lang.Thread.run(java.base@11.0.12/Thread.java:829)
{noformat}
Environment: (was: We encounter deadlock after failover - looks like order of locking in failover handling is different than locking order during rollback and thread are clashing:
Rollback:
- FifoMessageDispatchChannel
- then ActiveMQConnection
Failover handling:
- ActiveMQConnection
- then FifoMessageDispatchChannel
See the full stack traces:
{noformat}
"thread-doing-rollback":
at org.apache.activemq.ActiveMQConnection.getScheduler(ActiveMQConnection.java:2387)
- waiting to lock <0x00007f315a19d550> (a org.apache.activemq.ActiveMQConnection)
at org.apache.activemq.ActiveMQSession.getScheduler(ActiveMQSession.java:2254)
at org.apache.activemq.ActiveMQMessageConsumer.rollback(ActiveMQMessageConsumer.java:1356)
- locked <0x00007f37630c0d90> (a java.util.LinkedList)
- locked <0x00007f37630a4690> (a java.lang.Object)
at org.apache.activemq.ActiveMQMessageConsumer$5.afterRollback(ActiveMQMessageConsumer.java:1113)
at org.apache.activemq.TransactionContext.afterRollback(TransactionContext.java:166)
at org.apache.activemq.TransactionContext.commit(TransactionContext.java:344)
at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:583)
at org.apache.activemq.jms.pool.PooledSession.commit(PooledSession.java:167)
at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:218)
at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:776)
at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:680)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:331)
at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:270)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1237)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1227)
at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1120)
at java.lang.Thread.run(java.base@11.0.12/Thread.java:829)
"ActiveMQ Connection Executor: failover:(nio://....)&maxReconnectAttempts=0":
at org.apache.activemq.FifoMessageDispatchChannel.close(FifoMessageDispatchChannel.java:133)
- waiting to lock <0x00007f37630a4690> (a java.lang.Object)
at org.apache.activemq.ActiveMQMessageConsumer.dispose(ActiveMQMessageConsumer.java:872)
at org.apache.activemq.ActiveMQSession.dispose(ActiveMQSession.java:730)
- locked <0x00007f376311fd78> (a org.apache.activemq.ActiveMQSession)
at org.apache.activemq.ActiveMQConnection.close(ActiveMQConnection.java:671)
- locked <0x00007f315a19d550> (a org.apache.activemq.ActiveMQConnection)
at org.apache.activemq.jms.pool.ConnectionPool.close(ConnectionPool.java:176)
- locked <0x00007f315a230e50> (a org.apache.activemq.pool.PooledConnectionFactory$1)
at org.apache.activemq.jms.pool.ConnectionPool.onException(ConnectionPool.java:381)
at org.apache.activemq.ActiveMQConnection$5.run(ActiveMQConnection.java:1967)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.12/ThreadPoolExecutor.java:1128)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.12/ThreadPoolExecutor.java:628)
at java.lang.Thread.run(java.base@11.0.12/Thread.java:829)
{noformat}
)
> deadlock between threads calling failover and rollback
> ------------------------------------------------------
>
> Key: AMQ-9342
> URL: https://issues.apache.org/jira/browse/AMQ-9342
> Project: ActiveMQ
> Issue Type: Bug
> Reporter: bogdan t.
> Priority: Major
>
> We encounter deadlock after failover - looks like order of locking in failover handling is different than locking order during rollback and thread are clashing:
> Rollback:
> - FifoMessageDispatchChannel
> - then ActiveMQConnection
> Failover handling:
> - ActiveMQConnection
> - then FifoMessageDispatchChannel
> See the full stack traces:
> {noformat}
> "thread-doing-rollback":
> at org.apache.activemq.ActiveMQConnection.getScheduler(ActiveMQConnection.java:2387)
> - waiting to lock <0x00007f315a19d550> (a org.apache.activemq.ActiveMQConnection)
> at org.apache.activemq.ActiveMQSession.getScheduler(ActiveMQSession.java:2254)
> at org.apache.activemq.ActiveMQMessageConsumer.rollback(ActiveMQMessageConsumer.java:1356)
> - locked <0x00007f37630c0d90> (a java.util.LinkedList)
> - locked <0x00007f37630a4690> (a java.lang.Object)
> at org.apache.activemq.ActiveMQMessageConsumer$5.afterRollback(ActiveMQMessageConsumer.java:1113)
> at org.apache.activemq.TransactionContext.afterRollback(TransactionContext.java:166)
> at org.apache.activemq.TransactionContext.commit(TransactionContext.java:344)
> at org.apache.activemq.ActiveMQSession.commit(ActiveMQSession.java:583)
> at org.apache.activemq.jms.pool.PooledSession.commit(PooledSession.java:167)
> at org.springframework.jms.support.JmsUtils.commitIfNecessary(JmsUtils.java:218)
> at org.springframework.jms.listener.AbstractMessageListenerContainer.commitIfNecessary(AbstractMessageListenerContainer.java:776)
> at org.springframework.jms.listener.AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:680)
> at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:331)
> at org.springframework.jms.listener.AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:270)
> at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1237)
> at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1227)
> at org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:1120)
> at java.lang.Thread.run(java.base@11.0.12/Thread.java:829)
> "ActiveMQ Connection Executor: failover:(nio://....)&maxReconnectAttempts=0":
> at org.apache.activemq.FifoMessageDispatchChannel.close(FifoMessageDispatchChannel.java:133)
> - waiting to lock <0x00007f37630a4690> (a java.lang.Object)
> at org.apache.activemq.ActiveMQMessageConsumer.dispose(ActiveMQMessageConsumer.java:872)
> at org.apache.activemq.ActiveMQSession.dispose(ActiveMQSession.java:730)
> - locked <0x00007f376311fd78> (a org.apache.activemq.ActiveMQSession)
> at org.apache.activemq.ActiveMQConnection.close(ActiveMQConnection.java:671)
> - locked <0x00007f315a19d550> (a org.apache.activemq.ActiveMQConnection)
> at org.apache.activemq.jms.pool.ConnectionPool.close(ConnectionPool.java:176)
> - locked <0x00007f315a230e50> (a org.apache.activemq.pool.PooledConnectionFactory$1)
> at org.apache.activemq.jms.pool.ConnectionPool.onException(ConnectionPool.java:381)
> at org.apache.activemq.ActiveMQConnection$5.run(ActiveMQConnection.java:1967)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.12/ThreadPoolExecutor.java:1128)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.12/ThreadPoolExecutor.java:628)
> at java.lang.Thread.run(java.base@11.0.12/Thread.java:829)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)