You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@qpid.apache.org by Rajith Attapattu <ra...@gmail.com> on 2008/02/29 18:48:41 UTC

MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Hi All,

Following is the thread dump which identifies the deadlock.
Note this happens within the 0-10 code path.

This seems to have been introduced in revision 632331 after the fix for
QPID-823 in AMQSession.
If I rollback this deadlock doesn't happen.

[junit] Running org.apache.qpid.client.MessageListenerMultiConsumerTest
Full thread dump Java HotSpot(TM) Client VM (1.5.0_09-b03 mixed mode,
sharing):

"Dispatcher-Channel-2" daemon prio=1 tid=0x0aec0ba0 nid=0x362f waiting for
monitor entry [0xb047f000..0xb047f130]
        at org.apache.qpid.client.AMQSession$Dispatcher.run(AMQSession.java
:2449)
        - waiting to lock <0x88bab5a8> (a java.lang.Object)
        - locked <0x88be5810> (a java.lang.Object)

"SocketConnectorIoProcessor-4.0" prio=1 tid=0x0ae4a2a0 nid=0x362c runnable
[0xb037c000..0xb037ceb0]
        at sun.nio.ch.PollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.PollArrayWrapper.poll(PollArrayWrapper.java:100)
        at sun.nio.ch.PollSelectorImpl.doSelect(PollSelectorImpl.java:56)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
        - locked <0x88d905c0> (a sun.nio.ch.Util$1)
        - locked <0x88d9f0d8> (a java.util.Collections$UnmodifiableSet)
        - locked <0x88d9d568> (a sun.nio.ch.PollSelectorImpl)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
        at org.apache.mina.transport.socket.nio.SocketIoProcessor$Worker.run
(SocketIoProcessor.java:558)
        at org.apache.mina.util.NamePreservingRunnable.run(
NamePreservingRunnable.java:43)
        at java.lang.Thread.run(Thread.java:595)

"Thread-47" prio=1 tid=0x08d68e58 nid=0x361a runnable
[0xb0580000..0xb0580fb0]
        at java.io.FileInputStream.readBytes(Native Method)
        at java.io.FileInputStream.read(FileInputStream.java:194)
        at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
        at java.io.BufferedInputStream.read1(BufferedInputStream.java:256)
        at java.io.BufferedInputStream.read(BufferedInputStream.java:313)
        - locked <0x88d92810> (a java.io.BufferedInputStream)
        at java.io.FilterInputStream.read(FilterInputStream.java:90)
        at org.apache.qpid.testutil.QpidTestCase$Piper.run(QpidTestCase.java
:103)

"process reaper" daemon prio=1 tid=0x08e54828 nid=0x3618 runnable
[0xb0601000..0xb0602030]
        at java.lang.UNIXProcess.waitForProcessExit(Native Method)
        at java.lang.UNIXProcess.access$900(UNIXProcess.java:20)
        at java.lang.UNIXProcess$1$1.run(UNIXProcess.java:132)

"VmPipeIdleStatusChecker" daemon prio=1 tid=0x08db1438 nid=0x35e1 waiting on
condition [0xb0703000..0xb0703f30]
        at java.lang.Thread.sleep(Native Method)
        at
org.apache.mina.transport.vmpipe.support.VmPipeIdleStatusChecker$Worker.run(
VmPipeIdleStatusChecker.java:74)

"PooledByteBufferExpirer-0" daemon prio=1 tid=0x0887c408 nid=0x35dd waiting
on condition [0xb0f06000..0xb0f070b0]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.mina.common.PooledByteBufferAllocator$Expirer.run(
PooledByteBufferAllocator.java:277)

"AnonymousIoService-2" daemon prio=1 tid=0x08872a08 nid=0x35dc in
Object.wait() [0xb0a3d000..0xb0a3e130]
        at java.lang.Object.wait(Native Method)
        - waiting on <0x89696af8> (a
edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
)
        at java.lang.Object.wait(Object.java:474)
        at
edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue.take(
LinkedBlockingQueue.java:316)
        - locked <0x89696af8> (a
edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
)
        at
edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor.getTask(
ThreadPoolExecutor.java:493)
        at
edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run
(ThreadPoolExecutor.java:689)
        at java.lang.Thread.run(Thread.java:595)

"AnonymousIoService-1" daemon prio=1 tid=0x08841b10 nid=0x35db in
Object.wait() [0xb0abe000..0xb0abedb0]
        at java.lang.Object.wait(Native Method)
        - waiting on <0x89696af8> (a
edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
)
        at java.lang.Object.wait(Object.java:474)
        at
edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue.take(
LinkedBlockingQueue.java:316)
        - locked <0x89696af8> (a
edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
)
        at
edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor.getTask(
ThreadPoolExecutor.java:493)
        at
edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run
(ThreadPoolExecutor.java:689)
        at java.lang.Thread.run(Thread.java:595)

"SocketAcceptor-0" prio=1 tid=0x08d47028 nid=0x35d7 runnable
[0xb0d83000..0xb0d83fb0]
        at sun.nio.ch.PollArrayWrapper.poll0(Native Method)
        at sun.nio.ch.PollArrayWrapper.poll(PollArrayWrapper.java:100)
        at sun.nio.ch.PollSelectorImpl.doSelect(PollSelectorImpl.java:56)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
        - locked <0x89696c58> (a sun.nio.ch.Util$1)
        - locked <0x89696c68> (a java.util.Collections$UnmodifiableSet)
        - locked <0x89696c08> (a sun.nio.ch.PollSelectorImpl)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
        at org.apache.mina.transport.socket.nio.SocketAcceptor$Worker.run(
SocketAcceptor.java:266)
        at org.apache.mina.util.NamePreservingRunnable.run(
NamePreservingRunnable.java:43)
        at java.lang.Thread.run(Thread.java:595)

"Thread-5" daemon prio=1 tid=0x08cff508 nid=0x357b waiting for monitor entry
[0xb0e04000..0xb0e04db0]
        at java.io.BufferedInputStream.read(BufferedInputStream.java:304)
        - waiting to lock <0x89078838> (a java.io.BufferedInputStream)
        at java.io.FilterInputStream.read(FilterInputStream.java:111)
        at java.io.FilterInputStream.read(FilterInputStream.java:90)
        at org.apache.tools.ant.taskdefs.StreamPumper.run(StreamPumper.java
:92)
        at java.lang.Thread.run(Thread.java:595)

"Thread-2" daemon prio=1 tid=0x08cfb398 nid=0x356d runnable
[0xb0e85000..0xb0e86030]
        at java.io.FileInputStream.readBytes(Native Method)
        at java.io.FileInputStream.read(FileInputStream.java:194)
        at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
        at java.io.BufferedInputStream.read1(BufferedInputStream.java:256)
        at java.io.BufferedInputStream.read(BufferedInputStream.java:313)
        - locked <0x89078838> (a java.io.BufferedInputStream)
        at java.io.FilterInputStream.read(FilterInputStream.java:111)
        at java.io.FilterInputStream.read(FilterInputStream.java:90)
        at org.apache.tools.ant.taskdefs.StreamPumper.run(StreamPumper.java
:92)
        at java.lang.Thread.run(Thread.java:595)

"Low Memory Detector" daemon prio=1 tid=0x0880d5c8 nid=0x3567 runnable
[0x00000000..0x00000000]

"CompilerThread0" daemon prio=1 tid=0x0880c020 nid=0x3566 waiting on
condition [0x00000000..0xb22ef918]

"Signal Dispatcher" daemon prio=1 tid=0x0880b0d8 nid=0x3565 runnable
[0x00000000..0x00000000]

"Finalizer" daemon prio=1 tid=0x08804228 nid=0x3564 in Object.wait()
[0xb25f1000..0xb25f2030]
        at java.lang.Object.wait(Native Method)
        - waiting on <0x89075e68> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
        - locked <0x89075e68> (a java.lang.ref.ReferenceQueue$Lock)
        at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
        at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)

"Reference Handler" daemon prio=1 tid=0x08803528 nid=0x3563 in Object.wait()
[0xb2672000..0xb26730b0]
        at java.lang.Object.wait(Native Method)
        - waiting on <0x89075ee8> (a java.lang.ref.Reference$Lock)
        at java.lang.Object.wait(Object.java:474)
        at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
        - locked <0x89075ee8> (a java.lang.ref.Reference$Lock)

"main" prio=1 tid=0x087c4670 nid=0x3560 waiting for monitor entry
[0xbf913000..0xbf9145a8]
        at org.apache.qpid.client.AMQSession$Dispatcher.rejectPending(
AMQSession.java:2347)
        - waiting to lock <0x88be5810> (a java.lang.Object)
        at org.apache.qpid.client.AMQSession.confirmConsumerCancelled(
AMQSession.java:603)
        at org.apache.qpid.client.BasicMessageConsumer_0_10.sendCancel
(BasicMessageConsumer_0_10.java:209)
        at org.apache.qpid.client.BasicMessageConsumer.close(
BasicMessageConsumer.java:520)
        - locked <0x88d9dab8> (a java.lang.Object)
        at org.apache.qpid.client.BasicMessageConsumer.close(
BasicMessageConsumer.java:487)
        at org.apache.qpid.client.BasicMessageConsumer_0_10.close
(BasicMessageConsumer_0_10.java:487)
        at org.apache.qpid.client.AMQSession.closeConsumers(AMQSession.java
:1791)
        at org.apache.qpid.client.AMQSession.closeProducersAndConsumers(
AMQSession.java:1841)
        at org.apache.qpid.client.AMQSession.close(AMQSession.java:473)
        - locked <0x88d9dab8> (a java.lang.Object)
        - locked <0x88bab5a8> (a java.lang.Object)
        at org.apache.qpid.client.AMQConnection.closeAllSessions(
AMQConnection.java:754)
        at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
:673)
        - locked <0x88d9dab8> (a java.lang.Object)
        at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
:659)
        at org.apache.qpid.client.MessageListenerMultiConsumerTest.tearDown(
MessageListenerMultiConsumerTest.java:106)
        at junit.framework.TestCase.runBare(TestCase.java:130)
        at org.apache.qpid.testutil.QpidTestCase.runBare(QpidTestCase.java
:71)
        at junit.framework.TestResult$1.protect(TestResult.java:106)
        at junit.framework.TestResult.runProtected(TestResult.java:124)
        at junit.framework.TestResult.run(TestResult.java:109)
        at junit.framework.TestCase.run(TestCase.java:118)
        at junit.framework.TestSuite.runTest(TestSuite.java:208)
        at junit.framework.TestSuite.run(TestSuite.java:203)
        at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(
JUnitTestRunner.java:421)
        at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeInVM(
JUnitTask.java:1245)
        at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
JUnitTask.java:814)
        at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeOrQueue(
JUnitTask.java:1657)
        at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
JUnitTask.java:764)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:357)
        at org.apache.tools.ant.Target.performTasks(Target.java:385)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java
:1329)
        at org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(
SingleCheckExecutor.java:38)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
        at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:416)
        at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:289)
        at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:208)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.taskdefs.Sequential.execute(Sequential.java
:62)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.taskdefs.MacroInstance.execute(
MacroInstance.java:391)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:357)
        at org.apache.tools.ant.Target.performTasks(Target.java:385)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java
:1329)
        at org.apache.tools.ant.Project.executeTarget(Project.java:1298)
        at org.apache.tools.ant.helper.DefaultExecutor.executeTargets(
DefaultExecutor.java:41)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
        at org.apache.tools.ant.Main.runBuild(Main.java:698)
        at org.apache.tools.ant.Main.startAnt(Main.java:199)
        at org.apache.tools.ant.launch.Launcher.run(Launcher.java:257)
        at org.apache.tools.ant.launch.Launcher.main(Launcher.java:104)

"VM Thread" prio=1 tid=0x08800948 nid=0x3562 runnable

"VM Periodic Task Thread" prio=1 tid=0x0880eab0 nid=0x3568 waiting on
condition


Found one Java-level deadlock:
=============================
"Dispatcher-Channel-2":
  waiting to lock monitor 0x088052bc (object 0x88bab5a8, a java.lang.Object
),
  which is held by "main"
"main":
  waiting to lock monitor 0x0880533c (object 0x88be5810, a java.lang.Object
),
  which is held by "Dispatcher-Channel-2"

Java stack information for the threads listed above:
===================================================
"Dispatcher-Channel-2":
        at org.apache.qpid.client.AMQSession$Dispatcher.run(AMQSession.java
:2449)
        - waiting to lock <0x88bab5a8> (a java.lang.Object)
        - locked <0x88be5810> (a java.lang.Object)
"main":
        at org.apache.qpid.client.AMQSession$Dispatcher.rejectPending(
AMQSession.java:2347)
        - waiting to lock <0x88be5810> (a java.lang.Object)
        at org.apache.qpid.client.AMQSession.confirmConsumerCancelled(
AMQSession.java:603)
        at org.apache.qpid.client.BasicMessageConsumer_0_10.sendCancel
(BasicMessageConsumer_0_10.java:209)
        at org.apache.qpid.client.BasicMessageConsumer.close(
BasicMessageConsumer.java:520)
        - locked <0x88d9dab8> (a java.lang.Object)
        at org.apache.qpid.client.BasicMessageConsumer.close(
BasicMessageConsumer.java:487)
        at org.apache.qpid.client.BasicMessageConsumer_0_10.close
(BasicMessageConsumer_0_10.java:487)
        at org.apache.qpid.client.AMQSession.closeConsumers(AMQSession.java
:1791)
        at org.apache.qpid.client.AMQSession.closeProducersAndConsumers(
AMQSession.java:1841)
        at org.apache.qpid.client.AMQSession.close(AMQSession.java:473)
        - locked <0x88d9dab8> (a java.lang.Object)
        - locked <0x88bab5a8> (a java.lang.Object)
        at org.apache.qpid.client.AMQConnection.closeAllSessions(
AMQConnection.java:754)
        at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
:673)
        - locked <0x88d9dab8> (a java.lang.Object)
        at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
:659)
        at org.apache.qpid.client.MessageListenerMultiConsumerTest.tearDown(
MessageListenerMultiConsumerTest.java:106)
        at junit.framework.TestCase.runBare(TestCase.java:130)
        at org.apache.qpid.testutil.QpidTestCase.runBare(QpidTestCase.java
:71)
        at junit.framework.TestResult$1.protect(TestResult.java:106)
        at junit.framework.TestResult.runProtected(TestResult.java:124)
        at junit.framework.TestResult.run(TestResult.java:109)
        at junit.framework.TestCase.run(TestCase.java:118)
        at junit.framework.TestSuite.runTest(TestSuite.java:208)
        at junit.framework.TestSuite.run(TestSuite.java:203)
        at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(
JUnitTestRunner.java:421)
        at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeInVM(
JUnitTask.java:1245)
        at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
JUnitTask.java:814)
        at
org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeOrQueue(
JUnitTask.java:1657)
        at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
JUnitTask.java:764)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:357)
        at org.apache.tools.ant.Target.performTasks(Target.java:385)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java
:1329)
        at org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(
SingleCheckExecutor.java:38)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
        at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:416)
        at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:289)
        at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:208)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.taskdefs.Sequential.execute(Sequential.java
:62)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.taskdefs.MacroInstance.execute(
MacroInstance.java:391)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
:288)
        at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(
DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:585)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(
DispatchUtils.java:105)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:357)
        at org.apache.tools.ant.Target.performTasks(Target.java:385)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java
:1329)
        at org.apache.tools.ant.Project.executeTarget(Project.java:1298)
        at org.apache.tools.ant.helper.DefaultExecutor.executeTargets(
DefaultExecutor.java:41)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
        at org.apache.tools.ant.Main.runBuild(Main.java:698)
        at org.apache.tools.ant.Main.startAnt(Main.java:199)
        at org.apache.tools.ant.launch.Launcher.run(Launcher.java:257)
        at org.apache.tools.ant.launch.Launcher.main(Launcher.java:104)

Found 1 deadlock.
-- 
Regards,

Rajith Attapattu
Red Hat
blog: http://rajith.2rlabs.com/

Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Arnaud Simon <as...@redhat.com>.
On Mon, 2008-03-03 at 07:33 -0500, Rafael Schloming wrote:
> I thought this too, however on closer examination the closing thread 
> does a syncWrite, which means that thread will block until the 
> rejectPending is complete, so it should be the same as executing in
> the 
> same thread.

Yes, this is what I just said, the close operation will timeout. 



Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Arnaud Simon <as...@redhat.com>.
On Mon, 2008-03-03 at 07:33 -0500, Rafael Schloming wrote:
> I suspect the potential for the deadlock to occur is actually still 
> there on the 0-8 codepath, but it is not occuring for some reason. I 
> don't know if this is timing related or some other reason. The stack 
> trace suggests that the dispatcher thread is running as we attempt to 
> cancel the consumer. Stopping the thread should prevent the deadlock,
> I 
> don't know if the 0-8 code does that or not.
> 

It is not occurring when running MessageListenerMultiConsumerTest as all
the tests pass. With 0.10 there are some tests that we know should fail
until we have a way of turning pre-fetching off. I believe this is the
reason why we will not see this happening when using the 0.8 code path.
Wit the 0.10 code path messages are still in the process of being
dispatched when the test fails and call tearDown that closes the
connection. 


Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Rafael Schloming <ra...@redhat.com>.
The attached patch appears to fix the deadlock by merging _lock and 
_messageDeliveryLock into one. Unfortunately this needs to be 
conditional on 0-8 vs 0-10 codepath because merging the locks for 0-8 
causes the same deadlock that it prevents on 0-10 since on 0-8 
rejectPending is executed in another thread, and the closing thread 
blocks until that is complete.

I'm not entirely happy with this as a solution since I'd like to 
understand if the potential for deadlock is still there in the 0-8 code.

--Rafael

Rafael Schloming wrote:
> Arnaud Simon wrote:
>> On Mon, 2008-03-03 at 10:22 +0000, Aidan Skinner wrote:
>>> On Fri, Feb 29, 2008 at 8:23 PM, Rafael Schloming 
>>> <ra...@redhat.com> wrote:
>>>
>>>> The deadlock is between the session's _messageDeliveryLock and the
>>>>  Dispatcher's _lock. The dispatcher thread's main loop attempts to
>>>>  acquire _lock first and then _messageDeliveryLock. The main thread
>>>>  acquires _messageDeliveryLock first thing on close(...) and then
>>>>  subsequently many levels down the stack attempts to acquire _lock in
>>>>  order to call Dispatcher.rejectPending(...).
>>> We found (and fixed) several similar problems on M2, it's probably
>>> worth having a look at that code to make sure there aren't any cases
>>> being missed.
>>>
>>> - Aidan
>>>
>>
>> Rafael is absolutely right, Session.close is holding a lock on
>> _messageDeliveryLock only to prevent messages being delivered to
>> consumers when the session is closing (note that this lock should also
>> be held when consumer are closing, see Qpid-812). There is however no
>> reason for the dispatcher thread to keep a lock on _lock when
>> dispatching a message. _lock is used for stopping/starting the
>> dispatcher thread and _messageDeliveryLock for preventing message
>> delivery. So, we should change the run method of dispatcher for
>> releasing _lock and rollback and rejectPending for synchronizing on
>> _messageDeliveryLock. 
> 
> This was my first thought too, however I think there actually is a 
> reason. Connection.stop() is supposed to block until all message 
> listeners are finished with whatever they're currently work on. Holding 
> _lock while executing the message listener does accomplish this. If we 
> were to release the lock earlier then stop() could return while the 
> message listener is still in progress.
> 
>> Note that this deadlock situation would not happen with 0.8 as an
>> external thread is calling rejectPending but (if the dispatcher is
>> trying to dispatch a message) this thread will need to wait until close
>> finishes and will create a new dispatcher thread that will not be
>> closed.
> 
> I thought this too, however on closer examination the closing thread 
> does a syncWrite, which means that thread will block until the 
> rejectPending is complete, so it should be the same as executing in the 
> same thread.
> 
> I suspect the potential for the deadlock to occur is actually still 
> there on the 0-8 codepath, but it is not occuring for some reason. I 
> don't know if this is timing related or some other reason. The stack 
> trace suggests that the dispatcher thread is running as we attempt to 
> cancel the consumer. Stopping the thread should prevent the deadlock, I 
> don't know if the 0-8 code does that or not.
> 
>> The way the dispatcher thread is closed should also be changed. The
>> close operation currently interrupts the thread this means that the
>> dispatcher does not have a chance to acquire a lock on
>> _messageDeliveryLock for correctly releasing the message it may be
>> waiting to dispatch. 
> 
> Agreed, the way it checks a private _closed variable rather than 
> checking the session's _closed variable looks a bit dodgy also.
> 
> --Rafael

Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Rafael Schloming <ra...@redhat.com>.
Arnaud Simon wrote:
> On Mon, 2008-03-03 at 10:22 +0000, Aidan Skinner wrote:
>> On Fri, Feb 29, 2008 at 8:23 PM, Rafael Schloming <ra...@redhat.com> wrote:
>>
>>> The deadlock is between the session's _messageDeliveryLock and the
>>>  Dispatcher's _lock. The dispatcher thread's main loop attempts to
>>>  acquire _lock first and then _messageDeliveryLock. The main thread
>>>  acquires _messageDeliveryLock first thing on close(...) and then
>>>  subsequently many levels down the stack attempts to acquire _lock in
>>>  order to call Dispatcher.rejectPending(...).
>> We found (and fixed) several similar problems on M2, it's probably
>> worth having a look at that code to make sure there aren't any cases
>> being missed.
>>
>> - Aidan
>>
> 
> Rafael is absolutely right, Session.close is holding a lock on
> _messageDeliveryLock only to prevent messages being delivered to
> consumers when the session is closing (note that this lock should also
> be held when consumer are closing, see Qpid-812). There is however no
> reason for the dispatcher thread to keep a lock on _lock when
> dispatching a message. _lock is used for stopping/starting the
> dispatcher thread and _messageDeliveryLock for preventing message
> delivery. So, we should change the run method of dispatcher for
> releasing _lock and rollback and rejectPending for synchronizing on
> _messageDeliveryLock. 

This was my first thought too, however I think there actually is a 
reason. Connection.stop() is supposed to block until all message 
listeners are finished with whatever they're currently work on. Holding 
_lock while executing the message listener does accomplish this. If we 
were to release the lock earlier then stop() could return while the 
message listener is still in progress.

> Note that this deadlock situation would not happen with 0.8 as an
> external thread is calling rejectPending but (if the dispatcher is
> trying to dispatch a message) this thread will need to wait until close
> finishes and will create a new dispatcher thread that will not be
> closed.

I thought this too, however on closer examination the closing thread 
does a syncWrite, which means that thread will block until the 
rejectPending is complete, so it should be the same as executing in the 
same thread.

I suspect the potential for the deadlock to occur is actually still 
there on the 0-8 codepath, but it is not occuring for some reason. I 
don't know if this is timing related or some other reason. The stack 
trace suggests that the dispatcher thread is running as we attempt to 
cancel the consumer. Stopping the thread should prevent the deadlock, I 
don't know if the 0-8 code does that or not.

> The way the dispatcher thread is closed should also be changed. The
> close operation currently interrupts the thread this means that the
> dispatcher does not have a chance to acquire a lock on
> _messageDeliveryLock for correctly releasing the message it may be
> waiting to dispatch. 

Agreed, the way it checks a private _closed variable rather than 
checking the session's _closed variable looks a bit dodgy also.

--Rafael

Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Arnaud Simon <as...@redhat.com>.
On Mon, 2008-03-03 at 12:03 +0100, Arnaud Simon wrote:
> Rafael is absolutely right, Session.close is holding a lock on
> _messageDeliveryLock only to prevent messages being delivered to
> consumers when the session is closing (note that this lock should also
> be held when consumer are closing, see Qpid-812). There is however no
> reason for the dispatcher thread to keep a lock on _lock when
> dispatching a message. _lock is used for stopping/starting the
> dispatcher thread and _messageDeliveryLock for preventing message
> delivery. So, we should change the run method of dispatcher for
> releasing _lock and rollback and rejectPending for synchronizing on
> _messageDeliveryLock. 


> Note that this deadlock situation would not happen with 0.8 as an
> external thread is calling rejectPending but (if the dispatcher is
> trying to dispatch a message) this thread will need to wait until
> close
> finishes and will create a new dispatcher thread that will not be
> closed.

I was wrong actually the deadlock exists but the close operation will
eventually throw an exception as the synchronous operation sendCancel is
timing out. 


Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Arnaud Simon <as...@redhat.com>.
On Mon, 2008-03-03 at 10:22 +0000, Aidan Skinner wrote:
> On Fri, Feb 29, 2008 at 8:23 PM, Rafael Schloming <ra...@redhat.com> wrote:
> 
> > The deadlock is between the session's _messageDeliveryLock and the
> >  Dispatcher's _lock. The dispatcher thread's main loop attempts to
> >  acquire _lock first and then _messageDeliveryLock. The main thread
> >  acquires _messageDeliveryLock first thing on close(...) and then
> >  subsequently many levels down the stack attempts to acquire _lock in
> >  order to call Dispatcher.rejectPending(...).
> 
> We found (and fixed) several similar problems on M2, it's probably
> worth having a look at that code to make sure there aren't any cases
> being missed.
> 
> - Aidan
> 

Rafael is absolutely right, Session.close is holding a lock on
_messageDeliveryLock only to prevent messages being delivered to
consumers when the session is closing (note that this lock should also
be held when consumer are closing, see Qpid-812). There is however no
reason for the dispatcher thread to keep a lock on _lock when
dispatching a message. _lock is used for stopping/starting the
dispatcher thread and _messageDeliveryLock for preventing message
delivery. So, we should change the run method of dispatcher for
releasing _lock and rollback and rejectPending for synchronizing on
_messageDeliveryLock. 
Note that this deadlock situation would not happen with 0.8 as an
external thread is calling rejectPending but (if the dispatcher is
trying to dispatch a message) this thread will need to wait until close
finishes and will create a new dispatcher thread that will not be
closed.
The way the dispatcher thread is closed should also be changed. The
close operation currently interrupts the thread this means that the
dispatcher does not have a chance to acquire a lock on
_messageDeliveryLock for correctly releasing the message it may be
waiting to dispatch. 

Arnaud

 


Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Arnaud Simon <as...@redhat.com>.
I believe that the attached patch solves this issue for both 0.8 and
0.10. If we stop the dispatcher thread before closing the session it
should not acquire a lock on _messageDeliveryLock. This may be useful to
apply it on M2.1.1.

Arnaud 

On Mon, 2008-03-03 at 08:15 -0500, Rafael Schloming wrote: 
> Aidan Skinner wrote:
> > On Fri, Feb 29, 2008 at 8:23 PM, Rafael Schloming <ra...@redhat.com> wrote:
> > 
> >> The deadlock is between the session's _messageDeliveryLock and the
> >>  Dispatcher's _lock. The dispatcher thread's main loop attempts to
> >>  acquire _lock first and then _messageDeliveryLock. The main thread
> >>  acquires _messageDeliveryLock first thing on close(...) and then
> >>  subsequently many levels down the stack attempts to acquire _lock in
> >>  order to call Dispatcher.rejectPending(...).
> > 
> > We found (and fixed) several similar problems on M2, it's probably
> > worth having a look at that code to make sure there aren't any cases
> > being missed.
> 
> I've just examined the M2.1 code, and I think the potential for the same 
> deadlock exists there:
> 
> AMQSession.close(long timeout) acquires _messageDeliveryLock and then 
> indirectly calls BasicMessageConsumer.close().
> 
> BasicMessageConsumer.close() calls syncWrite(BasicCancelOkBody.class). 
> This will block until AMQSession.confirmConsumerCancelled() completes in 
> another thread.
> 
> If the dispatcher thread happens to be holding _lock and waiting to 
> acquire _messageDeliveryLock, the confirmConsumerCancelled() in the 
> other thread will never be able to acquire _lock and the same deadlock 
> should occur.
> 
> --Rafael
> 

Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Rafael Schloming <ra...@redhat.com>.
Aidan Skinner wrote:
> On Fri, Feb 29, 2008 at 8:23 PM, Rafael Schloming <ra...@redhat.com> wrote:
> 
>> The deadlock is between the session's _messageDeliveryLock and the
>>  Dispatcher's _lock. The dispatcher thread's main loop attempts to
>>  acquire _lock first and then _messageDeliveryLock. The main thread
>>  acquires _messageDeliveryLock first thing on close(...) and then
>>  subsequently many levels down the stack attempts to acquire _lock in
>>  order to call Dispatcher.rejectPending(...).
> 
> We found (and fixed) several similar problems on M2, it's probably
> worth having a look at that code to make sure there aren't any cases
> being missed.

I've just examined the M2.1 code, and I think the potential for the same 
deadlock exists there:

AMQSession.close(long timeout) acquires _messageDeliveryLock and then 
indirectly calls BasicMessageConsumer.close().

BasicMessageConsumer.close() calls syncWrite(BasicCancelOkBody.class). 
This will block until AMQSession.confirmConsumerCancelled() completes in 
another thread.

If the dispatcher thread happens to be holding _lock and waiting to 
acquire _messageDeliveryLock, the confirmConsumerCancelled() in the 
other thread will never be able to acquire _lock and the same deadlock 
should occur.

--Rafael


Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Aidan Skinner <ai...@apache.org>.
On Fri, Feb 29, 2008 at 8:23 PM, Rafael Schloming <ra...@redhat.com> wrote:

> The deadlock is between the session's _messageDeliveryLock and the
>  Dispatcher's _lock. The dispatcher thread's main loop attempts to
>  acquire _lock first and then _messageDeliveryLock. The main thread
>  acquires _messageDeliveryLock first thing on close(...) and then
>  subsequently many levels down the stack attempts to acquire _lock in
>  order to call Dispatcher.rejectPending(...).

We found (and fixed) several similar problems on M2, it's probably
worth having a look at that code to make sure there aren't any cases
being missed.

- Aidan

-- 
aim/y!:aidans42  g:aidan.skinner@gmail.com
http://aidan.skinner.me.uk/
"We belong to nobody and nobody belongs to us. We don't even belong to
each other."

Re: MessageListenerMultiConsumerTest deadlock when run in 0-10 code path using ant

Posted by Rafael Schloming <ra...@redhat.com>.
The deadlock is between the session's _messageDeliveryLock and the 
Dispatcher's _lock. The dispatcher thread's main loop attempts to 
acquire _lock first and then _messageDeliveryLock. The main thread 
acquires _messageDeliveryLock first thing on close(...) and then 
subsequently many levels down the stack attempts to acquire _lock in 
order to call Dispatcher.rejectPending(...).

The change causes this to occur more frequently because it moves the 
shutdown of the dispatcher thread to a later point, thereby increasing 
the probability a deadlock will occur, but I think the possibility was 
always there.

I believe the reason the 0-8 code path is not affected is because it 
calls AMQSession.confirmConsumerCanceled(...) in the 
BasicCancelOkMethodHandler which is executed in another thread, so the 
main thread doesn't try to acquire the Dispatcher's _lock.

--Rafael

Rajith Attapattu wrote:
> Hi All,
> 
> Following is the thread dump which identifies the deadlock.
> Note this happens within the 0-10 code path.
> 
> This seems to have been introduced in revision 632331 after the fix for
> QPID-823 in AMQSession.
> If I rollback this deadlock doesn't happen.
> 
> [junit] Running org.apache.qpid.client.MessageListenerMultiConsumerTest
> Full thread dump Java HotSpot(TM) Client VM (1.5.0_09-b03 mixed mode,
> sharing):
> 
> "Dispatcher-Channel-2" daemon prio=1 tid=0x0aec0ba0 nid=0x362f waiting for
> monitor entry [0xb047f000..0xb047f130]
>         at org.apache.qpid.client.AMQSession$Dispatcher.run(AMQSession.java
> :2449)
>         - waiting to lock <0x88bab5a8> (a java.lang.Object)
>         - locked <0x88be5810> (a java.lang.Object)
> 
> "SocketConnectorIoProcessor-4.0" prio=1 tid=0x0ae4a2a0 nid=0x362c runnable
> [0xb037c000..0xb037ceb0]
>         at sun.nio.ch.PollArrayWrapper.poll0(Native Method)
>         at sun.nio.ch.PollArrayWrapper.poll(PollArrayWrapper.java:100)
>         at sun.nio.ch.PollSelectorImpl.doSelect(PollSelectorImpl.java:56)
>         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
>         - locked <0x88d905c0> (a sun.nio.ch.Util$1)
>         - locked <0x88d9f0d8> (a java.util.Collections$UnmodifiableSet)
>         - locked <0x88d9d568> (a sun.nio.ch.PollSelectorImpl)
>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
>         at org.apache.mina.transport.socket.nio.SocketIoProcessor$Worker.run
> (SocketIoProcessor.java:558)
>         at org.apache.mina.util.NamePreservingRunnable.run(
> NamePreservingRunnable.java:43)
>         at java.lang.Thread.run(Thread.java:595)
> 
> "Thread-47" prio=1 tid=0x08d68e58 nid=0x361a runnable
> [0xb0580000..0xb0580fb0]
>         at java.io.FileInputStream.readBytes(Native Method)
>         at java.io.FileInputStream.read(FileInputStream.java:194)
>         at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
>         at java.io.BufferedInputStream.read1(BufferedInputStream.java:256)
>         at java.io.BufferedInputStream.read(BufferedInputStream.java:313)
>         - locked <0x88d92810> (a java.io.BufferedInputStream)
>         at java.io.FilterInputStream.read(FilterInputStream.java:90)
>         at org.apache.qpid.testutil.QpidTestCase$Piper.run(QpidTestCase.java
> :103)
> 
> "process reaper" daemon prio=1 tid=0x08e54828 nid=0x3618 runnable
> [0xb0601000..0xb0602030]
>         at java.lang.UNIXProcess.waitForProcessExit(Native Method)
>         at java.lang.UNIXProcess.access$900(UNIXProcess.java:20)
>         at java.lang.UNIXProcess$1$1.run(UNIXProcess.java:132)
> 
> "VmPipeIdleStatusChecker" daemon prio=1 tid=0x08db1438 nid=0x35e1 waiting on
> condition [0xb0703000..0xb0703f30]
>         at java.lang.Thread.sleep(Native Method)
>         at
> org.apache.mina.transport.vmpipe.support.VmPipeIdleStatusChecker$Worker.run(
> VmPipeIdleStatusChecker.java:74)
> 
> "PooledByteBufferExpirer-0" daemon prio=1 tid=0x0887c408 nid=0x35dd waiting
> on condition [0xb0f06000..0xb0f070b0]
>         at java.lang.Thread.sleep(Native Method)
>         at org.apache.mina.common.PooledByteBufferAllocator$Expirer.run(
> PooledByteBufferAllocator.java:277)
> 
> "AnonymousIoService-2" daemon prio=1 tid=0x08872a08 nid=0x35dc in
> Object.wait() [0xb0a3d000..0xb0a3e130]
>         at java.lang.Object.wait(Native Method)
>         - waiting on <0x89696af8> (a
> edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
> )
>         at java.lang.Object.wait(Object.java:474)
>         at
> edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue.take(
> LinkedBlockingQueue.java:316)
>         - locked <0x89696af8> (a
> edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
> )
>         at
> edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor.getTask(
> ThreadPoolExecutor.java:493)
>         at
> edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run
> (ThreadPoolExecutor.java:689)
>         at java.lang.Thread.run(Thread.java:595)
> 
> "AnonymousIoService-1" daemon prio=1 tid=0x08841b10 nid=0x35db in
> Object.wait() [0xb0abe000..0xb0abedb0]
>         at java.lang.Object.wait(Native Method)
>         - waiting on <0x89696af8> (a
> edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
> )
>         at java.lang.Object.wait(Object.java:474)
>         at
> edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue.take(
> LinkedBlockingQueue.java:316)
>         - locked <0x89696af8> (a
> edu.emory.mathcs.backport.java.util.concurrent.LinkedBlockingQueue$SerializableLock
> )
>         at
> edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor.getTask(
> ThreadPoolExecutor.java:493)
>         at
> edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run
> (ThreadPoolExecutor.java:689)
>         at java.lang.Thread.run(Thread.java:595)
> 
> "SocketAcceptor-0" prio=1 tid=0x08d47028 nid=0x35d7 runnable
> [0xb0d83000..0xb0d83fb0]
>         at sun.nio.ch.PollArrayWrapper.poll0(Native Method)
>         at sun.nio.ch.PollArrayWrapper.poll(PollArrayWrapper.java:100)
>         at sun.nio.ch.PollSelectorImpl.doSelect(PollSelectorImpl.java:56)
>         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
>         - locked <0x89696c58> (a sun.nio.ch.Util$1)
>         - locked <0x89696c68> (a java.util.Collections$UnmodifiableSet)
>         - locked <0x89696c08> (a sun.nio.ch.PollSelectorImpl)
>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
>         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
>         at org.apache.mina.transport.socket.nio.SocketAcceptor$Worker.run(
> SocketAcceptor.java:266)
>         at org.apache.mina.util.NamePreservingRunnable.run(
> NamePreservingRunnable.java:43)
>         at java.lang.Thread.run(Thread.java:595)
> 
> "Thread-5" daemon prio=1 tid=0x08cff508 nid=0x357b waiting for monitor entry
> [0xb0e04000..0xb0e04db0]
>         at java.io.BufferedInputStream.read(BufferedInputStream.java:304)
>         - waiting to lock <0x89078838> (a java.io.BufferedInputStream)
>         at java.io.FilterInputStream.read(FilterInputStream.java:111)
>         at java.io.FilterInputStream.read(FilterInputStream.java:90)
>         at org.apache.tools.ant.taskdefs.StreamPumper.run(StreamPumper.java
> :92)
>         at java.lang.Thread.run(Thread.java:595)
> 
> "Thread-2" daemon prio=1 tid=0x08cfb398 nid=0x356d runnable
> [0xb0e85000..0xb0e86030]
>         at java.io.FileInputStream.readBytes(Native Method)
>         at java.io.FileInputStream.read(FileInputStream.java:194)
>         at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
>         at java.io.BufferedInputStream.read1(BufferedInputStream.java:256)
>         at java.io.BufferedInputStream.read(BufferedInputStream.java:313)
>         - locked <0x89078838> (a java.io.BufferedInputStream)
>         at java.io.FilterInputStream.read(FilterInputStream.java:111)
>         at java.io.FilterInputStream.read(FilterInputStream.java:90)
>         at org.apache.tools.ant.taskdefs.StreamPumper.run(StreamPumper.java
> :92)
>         at java.lang.Thread.run(Thread.java:595)
> 
> "Low Memory Detector" daemon prio=1 tid=0x0880d5c8 nid=0x3567 runnable
> [0x00000000..0x00000000]
> 
> "CompilerThread0" daemon prio=1 tid=0x0880c020 nid=0x3566 waiting on
> condition [0x00000000..0xb22ef918]
> 
> "Signal Dispatcher" daemon prio=1 tid=0x0880b0d8 nid=0x3565 runnable
> [0x00000000..0x00000000]
> 
> "Finalizer" daemon prio=1 tid=0x08804228 nid=0x3564 in Object.wait()
> [0xb25f1000..0xb25f2030]
>         at java.lang.Object.wait(Native Method)
>         - waiting on <0x89075e68> (a java.lang.ref.ReferenceQueue$Lock)
>         at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:116)
>         - locked <0x89075e68> (a java.lang.ref.ReferenceQueue$Lock)
>         at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:132)
>         at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
> 
> "Reference Handler" daemon prio=1 tid=0x08803528 nid=0x3563 in Object.wait()
> [0xb2672000..0xb26730b0]
>         at java.lang.Object.wait(Native Method)
>         - waiting on <0x89075ee8> (a java.lang.ref.Reference$Lock)
>         at java.lang.Object.wait(Object.java:474)
>         at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
>         - locked <0x89075ee8> (a java.lang.ref.Reference$Lock)
> 
> "main" prio=1 tid=0x087c4670 nid=0x3560 waiting for monitor entry
> [0xbf913000..0xbf9145a8]
>         at org.apache.qpid.client.AMQSession$Dispatcher.rejectPending(
> AMQSession.java:2347)
>         - waiting to lock <0x88be5810> (a java.lang.Object)
>         at org.apache.qpid.client.AMQSession.confirmConsumerCancelled(
> AMQSession.java:603)
>         at org.apache.qpid.client.BasicMessageConsumer_0_10.sendCancel
> (BasicMessageConsumer_0_10.java:209)
>         at org.apache.qpid.client.BasicMessageConsumer.close(
> BasicMessageConsumer.java:520)
>         - locked <0x88d9dab8> (a java.lang.Object)
>         at org.apache.qpid.client.BasicMessageConsumer.close(
> BasicMessageConsumer.java:487)
>         at org.apache.qpid.client.BasicMessageConsumer_0_10.close
> (BasicMessageConsumer_0_10.java:487)
>         at org.apache.qpid.client.AMQSession.closeConsumers(AMQSession.java
> :1791)
>         at org.apache.qpid.client.AMQSession.closeProducersAndConsumers(
> AMQSession.java:1841)
>         at org.apache.qpid.client.AMQSession.close(AMQSession.java:473)
>         - locked <0x88d9dab8> (a java.lang.Object)
>         - locked <0x88bab5a8> (a java.lang.Object)
>         at org.apache.qpid.client.AMQConnection.closeAllSessions(
> AMQConnection.java:754)
>         at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
> :673)
>         - locked <0x88d9dab8> (a java.lang.Object)
>         at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
> :659)
>         at org.apache.qpid.client.MessageListenerMultiConsumerTest.tearDown(
> MessageListenerMultiConsumerTest.java:106)
>         at junit.framework.TestCase.runBare(TestCase.java:130)
>         at org.apache.qpid.testutil.QpidTestCase.runBare(QpidTestCase.java
> :71)
>         at junit.framework.TestResult$1.protect(TestResult.java:106)
>         at junit.framework.TestResult.runProtected(TestResult.java:124)
>         at junit.framework.TestResult.run(TestResult.java:109)
>         at junit.framework.TestCase.run(TestCase.java:118)
>         at junit.framework.TestSuite.runTest(TestSuite.java:208)
>         at junit.framework.TestSuite.run(TestSuite.java:203)
>         at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(
> JUnitTestRunner.java:421)
>         at
> org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeInVM(
> JUnitTask.java:1245)
>         at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
> JUnitTask.java:814)
>         at
> org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeOrQueue(
> JUnitTask.java:1657)
>         at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
> JUnitTask.java:764)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.Target.execute(Target.java:357)
>         at org.apache.tools.ant.Target.performTasks(Target.java:385)
>         at org.apache.tools.ant.Project.executeSortedTargets(Project.java
> :1329)
>         at org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(
> SingleCheckExecutor.java:38)
>         at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
>         at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:416)
>         at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:289)
>         at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:208)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.taskdefs.Sequential.execute(Sequential.java
> :62)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.taskdefs.MacroInstance.execute(
> MacroInstance.java:391)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.Target.execute(Target.java:357)
>         at org.apache.tools.ant.Target.performTasks(Target.java:385)
>         at org.apache.tools.ant.Project.executeSortedTargets(Project.java
> :1329)
>         at org.apache.tools.ant.Project.executeTarget(Project.java:1298)
>         at org.apache.tools.ant.helper.DefaultExecutor.executeTargets(
> DefaultExecutor.java:41)
>         at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
>         at org.apache.tools.ant.Main.runBuild(Main.java:698)
>         at org.apache.tools.ant.Main.startAnt(Main.java:199)
>         at org.apache.tools.ant.launch.Launcher.run(Launcher.java:257)
>         at org.apache.tools.ant.launch.Launcher.main(Launcher.java:104)
> 
> "VM Thread" prio=1 tid=0x08800948 nid=0x3562 runnable
> 
> "VM Periodic Task Thread" prio=1 tid=0x0880eab0 nid=0x3568 waiting on
> condition
> 
> 
> Found one Java-level deadlock:
> =============================
> "Dispatcher-Channel-2":
>   waiting to lock monitor 0x088052bc (object 0x88bab5a8, a java.lang.Object
> ),
>   which is held by "main"
> "main":
>   waiting to lock monitor 0x0880533c (object 0x88be5810, a java.lang.Object
> ),
>   which is held by "Dispatcher-Channel-2"
> 
> Java stack information for the threads listed above:
> ===================================================
> "Dispatcher-Channel-2":
>         at org.apache.qpid.client.AMQSession$Dispatcher.run(AMQSession.java
> :2449)
>         - waiting to lock <0x88bab5a8> (a java.lang.Object)
>         - locked <0x88be5810> (a java.lang.Object)
> "main":
>         at org.apache.qpid.client.AMQSession$Dispatcher.rejectPending(
> AMQSession.java:2347)
>         - waiting to lock <0x88be5810> (a java.lang.Object)
>         at org.apache.qpid.client.AMQSession.confirmConsumerCancelled(
> AMQSession.java:603)
>         at org.apache.qpid.client.BasicMessageConsumer_0_10.sendCancel
> (BasicMessageConsumer_0_10.java:209)
>         at org.apache.qpid.client.BasicMessageConsumer.close(
> BasicMessageConsumer.java:520)
>         - locked <0x88d9dab8> (a java.lang.Object)
>         at org.apache.qpid.client.BasicMessageConsumer.close(
> BasicMessageConsumer.java:487)
>         at org.apache.qpid.client.BasicMessageConsumer_0_10.close
> (BasicMessageConsumer_0_10.java:487)
>         at org.apache.qpid.client.AMQSession.closeConsumers(AMQSession.java
> :1791)
>         at org.apache.qpid.client.AMQSession.closeProducersAndConsumers(
> AMQSession.java:1841)
>         at org.apache.qpid.client.AMQSession.close(AMQSession.java:473)
>         - locked <0x88d9dab8> (a java.lang.Object)
>         - locked <0x88bab5a8> (a java.lang.Object)
>         at org.apache.qpid.client.AMQConnection.closeAllSessions(
> AMQConnection.java:754)
>         at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
> :673)
>         - locked <0x88d9dab8> (a java.lang.Object)
>         at org.apache.qpid.client.AMQConnection.close(AMQConnection.java
> :659)
>         at org.apache.qpid.client.MessageListenerMultiConsumerTest.tearDown(
> MessageListenerMultiConsumerTest.java:106)
>         at junit.framework.TestCase.runBare(TestCase.java:130)
>         at org.apache.qpid.testutil.QpidTestCase.runBare(QpidTestCase.java
> :71)
>         at junit.framework.TestResult$1.protect(TestResult.java:106)
>         at junit.framework.TestResult.runProtected(TestResult.java:124)
>         at junit.framework.TestResult.run(TestResult.java:109)
>         at junit.framework.TestCase.run(TestCase.java:118)
>         at junit.framework.TestSuite.runTest(TestSuite.java:208)
>         at junit.framework.TestSuite.run(TestSuite.java:203)
>         at org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner.run(
> JUnitTestRunner.java:421)
>         at
> org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeInVM(
> JUnitTask.java:1245)
>         at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
> JUnitTask.java:814)
>         at
> org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.executeOrQueue(
> JUnitTask.java:1657)
>         at org.apache.tools.ant.taskdefs.optional.junit.JUnitTask.execute(
> JUnitTask.java:764)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.Target.execute(Target.java:357)
>         at org.apache.tools.ant.Target.performTasks(Target.java:385)
>         at org.apache.tools.ant.Project.executeSortedTargets(Project.java
> :1329)
>         at org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(
> SingleCheckExecutor.java:38)
>         at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
>         at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:416)
>         at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:289)
>         at org.apache.tools.ant.taskdefs.SubAnt.execute(SubAnt.java:208)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.taskdefs.Sequential.execute(Sequential.java
> :62)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.taskdefs.MacroInstance.execute(
> MacroInstance.java:391)
>         at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java
> :288)
>         at sun.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:25)
>         at java.lang.reflect.Method.invoke(Method.java:585)
>         at org.apache.tools.ant.dispatch.DispatchUtils.execute(
> DispatchUtils.java:105)
>         at org.apache.tools.ant.Task.perform(Task.java:348)
>         at org.apache.tools.ant.Target.execute(Target.java:357)
>         at org.apache.tools.ant.Target.performTasks(Target.java:385)
>         at org.apache.tools.ant.Project.executeSortedTargets(Project.java
> :1329)
>         at org.apache.tools.ant.Project.executeTarget(Project.java:1298)
>         at org.apache.tools.ant.helper.DefaultExecutor.executeTargets(
> DefaultExecutor.java:41)
>         at org.apache.tools.ant.Project.executeTargets(Project.java:1181)
>         at org.apache.tools.ant.Main.runBuild(Main.java:698)
>         at org.apache.tools.ant.Main.startAnt(Main.java:199)
>         at org.apache.tools.ant.launch.Launcher.run(Launcher.java:257)
>         at org.apache.tools.ant.launch.Launcher.main(Launcher.java:104)
> 
> Found 1 deadlock.