You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ignite.apache.org by "Vitaliy Biryukov (JIRA)" <ji...@apache.org> on 2018/08/14 15:39:00 UTC
[jira] [Updated] (IGNITE-9267) Deadlock between unsuccessful client
reconnecting and stopping.
[ https://issues.apache.org/jira/browse/IGNITE-9267?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Vitaliy Biryukov updated IGNITE-9267:
-------------------------------------
Description:
*T1* start reconnecting and acquire *stateMux* (ZookeeperDiscoveryImpl#processLocalJoin) -> *T2* start stopping and acquire *IgnitionEx.this* (IgnitionEx#stop0) -> *T2* stopping ZookeeperDiscoveryImpl and wait for *stateMux* -> *T1* fails on reconnecting and start to stopping node and wait for *IgnitionEx.this*.
Reconnecting thread:
{noformat}
"zk-internal.IgniteClientReconnectCacheTest3-EventThread" #593633 daemon prio=5 os_prio=0 tid=0x00007ff8e4063800 nid=0x478e waiting for monitor entry [0x00007ff90f2f0000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2565)
- waiting to lock <0x00000000e9429280> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2557)
at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:374)
at org.apache.ignite.Ignition.stop(Ignition.java:229)
at org.apache.ignite.internal.IgniteKernal.close(IgniteKernal.java:3417)
at org.apache.ignite.internal.IgniteKernal.onReconnected(IgniteKernal.java:3904)
at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery0(GridDiscoveryManager.java:831)
at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery(GridDiscoveryManager.java:590)
- locked <0x00000000e9429468> (a java.lang.Object)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processLocalJoin(ZookeeperDiscoveryImpl.java:2960)
- locked <0x00000000e9429478> (a java.lang.Object)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processBulkJoin(ZookeeperDiscoveryImpl.java:2760)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processNewEvents(ZookeeperDiscoveryImpl.java:2623)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processNewEvents(ZookeeperDiscoveryImpl.java:2598)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.access$2000(ZookeeperDiscoveryImpl.java:108)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl$ZkWatcher.processResult(ZookeeperDiscoveryImpl.java:4108)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperClient$DataCallbackWrapper.processResult(ZookeeperClient.java:1219)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:561)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
{noformat}
Stopping thread:
{noformat}
"main" #1 prio=5 os_prio=0 tid=0x00007ffba000e000 nid=0x6aa3 waiting for monitor entry [0x00007ffba8c83000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.stop0(ZookeeperDiscoveryImpl.java:3838)
- waiting to lock <0x00000000e9429478> (a java.lang.Object)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.stop(ZookeeperDiscoveryImpl.java:3813)
at org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi.spiStop(ZookeeperDiscoverySpi.java:501)
at org.apache.ignite.internal.managers.GridManagerAdapter.stopSpi(GridManagerAdapter.java:330)
at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.stop(GridDiscoveryManager.java:1683)
at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2206)
at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2081)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2594)
- locked <0x00000000e9429280> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2557)
at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:374)
at org.apache.ignite.Ignition.stop(Ignition.java:229)
at org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153)
at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193)
at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174)
at org.apache.ignite.internal.IgniteClientReconnectCacheTest.afterTest(IgniteClientReconnectCacheTest.java:151)
at org.apache.ignite.testframework.junits.GridAbstractTest.tearDown(GridAbstractTest.java:1763)
at org.apache.ignite.testframework.junits.common.GridCommonAbstractTest.tearDown(GridCommonAbstractTest.java:503)
{noformat}
was:
Reconnecting thread:
{noformat}
"zk-internal.IgniteClientReconnectCacheTest3-EventThread" #593633 daemon prio=5 os_prio=0 tid=0x00007ff8e4063800 nid=0x478e waiting for monitor entry [0x00007ff90f2f0000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2565)
- waiting to lock <0x00000000e9429280> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2557)
at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:374)
at org.apache.ignite.Ignition.stop(Ignition.java:229)
at org.apache.ignite.internal.IgniteKernal.close(IgniteKernal.java:3417)
at org.apache.ignite.internal.IgniteKernal.onReconnected(IgniteKernal.java:3904)
at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery0(GridDiscoveryManager.java:831)
at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery(GridDiscoveryManager.java:590)
- locked <0x00000000e9429468> (a java.lang.Object)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processLocalJoin(ZookeeperDiscoveryImpl.java:2960)
- locked <0x00000000e9429478> (a java.lang.Object)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processBulkJoin(ZookeeperDiscoveryImpl.java:2760)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processNewEvents(ZookeeperDiscoveryImpl.java:2623)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processNewEvents(ZookeeperDiscoveryImpl.java:2598)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.access$2000(ZookeeperDiscoveryImpl.java:108)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl$ZkWatcher.processResult(ZookeeperDiscoveryImpl.java:4108)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperClient$DataCallbackWrapper.processResult(ZookeeperClient.java:1219)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:561)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
{noformat}
Stopping thread:
{noformat}
"main" #1 prio=5 os_prio=0 tid=0x00007ffba000e000 nid=0x6aa3 waiting for monitor entry [0x00007ffba8c83000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.stop0(ZookeeperDiscoveryImpl.java:3838)
- waiting to lock <0x00000000e9429478> (a java.lang.Object)
at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.stop(ZookeeperDiscoveryImpl.java:3813)
at org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi.spiStop(ZookeeperDiscoverySpi.java:501)
at org.apache.ignite.internal.managers.GridManagerAdapter.stopSpi(GridManagerAdapter.java:330)
at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.stop(GridDiscoveryManager.java:1683)
at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2206)
at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2081)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2594)
- locked <0x00000000e9429280> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2557)
at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:374)
at org.apache.ignite.Ignition.stop(Ignition.java:229)
at org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153)
at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193)
at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174)
at org.apache.ignite.internal.IgniteClientReconnectCacheTest.afterTest(IgniteClientReconnectCacheTest.java:151)
at org.apache.ignite.testframework.junits.GridAbstractTest.tearDown(GridAbstractTest.java:1763)
at org.apache.ignite.testframework.junits.common.GridCommonAbstractTest.tearDown(GridCommonAbstractTest.java:503)
{noformat}
> Deadlock between unsuccessful client reconnecting and stopping.
> ---------------------------------------------------------------
>
> Key: IGNITE-9267
> URL: https://issues.apache.org/jira/browse/IGNITE-9267
> Project: Ignite
> Issue Type: Bug
> Affects Versions: 2.6
> Reporter: Vitaliy Biryukov
> Priority: Major
> Fix For: 2.7
>
>
> *T1* start reconnecting and acquire *stateMux* (ZookeeperDiscoveryImpl#processLocalJoin) -> *T2* start stopping and acquire *IgnitionEx.this* (IgnitionEx#stop0) -> *T2* stopping ZookeeperDiscoveryImpl and wait for *stateMux* -> *T1* fails on reconnecting and start to stopping node and wait for *IgnitionEx.this*.
> Reconnecting thread:
> {noformat}
> "zk-internal.IgniteClientReconnectCacheTest3-EventThread" #593633 daemon prio=5 os_prio=0 tid=0x00007ff8e4063800 nid=0x478e waiting for monitor entry [0x00007ff90f2f0000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2565)
> - waiting to lock <0x00000000e9429280> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
> at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2557)
> at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:374)
> at org.apache.ignite.Ignition.stop(Ignition.java:229)
> at org.apache.ignite.internal.IgniteKernal.close(IgniteKernal.java:3417)
> at org.apache.ignite.internal.IgniteKernal.onReconnected(IgniteKernal.java:3904)
> at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery0(GridDiscoveryManager.java:831)
> at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4.onDiscovery(GridDiscoveryManager.java:590)
> - locked <0x00000000e9429468> (a java.lang.Object)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processLocalJoin(ZookeeperDiscoveryImpl.java:2960)
> - locked <0x00000000e9429478> (a java.lang.Object)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processBulkJoin(ZookeeperDiscoveryImpl.java:2760)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processNewEvents(ZookeeperDiscoveryImpl.java:2623)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.processNewEvents(ZookeeperDiscoveryImpl.java:2598)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.access$2000(ZookeeperDiscoveryImpl.java:108)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl$ZkWatcher.processResult(ZookeeperDiscoveryImpl.java:4108)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperClient$DataCallbackWrapper.processResult(ZookeeperClient.java:1219)
> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:561)
> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
> {noformat}
> Stopping thread:
> {noformat}
> "main" #1 prio=5 os_prio=0 tid=0x00007ffba000e000 nid=0x6aa3 waiting for monitor entry [0x00007ffba8c83000]
> java.lang.Thread.State: BLOCKED (on object monitor)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.stop0(ZookeeperDiscoveryImpl.java:3838)
> - waiting to lock <0x00000000e9429478> (a java.lang.Object)
> at org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.stop(ZookeeperDiscoveryImpl.java:3813)
> at org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi.spiStop(ZookeeperDiscoverySpi.java:501)
> at org.apache.ignite.internal.managers.GridManagerAdapter.stopSpi(GridManagerAdapter.java:330)
> at org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.stop(GridDiscoveryManager.java:1683)
> at org.apache.ignite.internal.IgniteKernal.stop0(IgniteKernal.java:2206)
> at org.apache.ignite.internal.IgniteKernal.stop(IgniteKernal.java:2081)
> at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop0(IgnitionEx.java:2594)
> - locked <0x00000000e9429280> (a org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
> at org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.stop(IgnitionEx.java:2557)
> at org.apache.ignite.internal.IgnitionEx.stop(IgnitionEx.java:374)
> at org.apache.ignite.Ignition.stop(Ignition.java:229)
> at org.apache.ignite.testframework.junits.GridAbstractTest.stopGrid(GridAbstractTest.java:1153)
> at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1193)
> at org.apache.ignite.testframework.junits.GridAbstractTest.stopAllGrids(GridAbstractTest.java:1174)
> at org.apache.ignite.internal.IgniteClientReconnectCacheTest.afterTest(IgniteClientReconnectCacheTest.java:151)
> at org.apache.ignite.testframework.junits.GridAbstractTest.tearDown(GridAbstractTest.java:1763)
> at org.apache.ignite.testframework.junits.common.GridCommonAbstractTest.tearDown(GridCommonAbstractTest.java:503)
> {noformat}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)