You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@ignite.apache.org by VeenaMithare <v....@cmcmarkets.com> on 2021/01/11 14:54:56 UTC
Client stuck on startup
HI ,
One of the clients on my windows box sometimes gets stuck up on startup . I
see this in the thread dump of the client.
a. Not sure what those locked synchronizers are
"main" #1 prio=5 os_prio=0 tid=0x0000000002c9c800 nid=0x4404 waiting on
condition [0x0000000002c8e000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000006fda12d20> (a
java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at
org.apache.ignite.spi.discovery.tcp.ClientImpl.spiStart(ClientImpl.java:312)
at
org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:2099)
at
org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:299)
at
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:943)
at
org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1960)
at org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:1276)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:2045)
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1703)
- locked <0x00000006db5ca060> (a
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
at org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1117)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:637)
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:563)
at org.apache.ignite.Ignition.start(Ignition.java:321)
at
com.company.prophet.projectname.dao.projectnameClientUtil.startIgnite(projectnameClientUtil.java:168)
at
com.company.prophet.projectname.dao.projectnameClientUtil.startIgniteForSingleOrBlueGreen(projectnameClientUtil.java:109)
at
com.company.prophet.projectname.dao.projectnameClientUtil.startIgnite(projectnameClientUtil.java:84)
at
com.company.prophet.projectname.dao.projectnameClient.start(projectnameClient.java:126)
at
com.company.prophet.projectname.sync.main.Sync.startIgnite(Sync.java:264)
at com.company.prophet.projectname.sync.main.Sync.start(Sync.java:133)
at
com.company.prophet.milliways.shared.common.launcher.MilliwaysGenericLauncher.run(MilliwaysGenericLauncher.java:69)
at
com.company.prophet.milliways.shared.common.launcher.MilliwaysGenericLauncher$1.run(MilliwaysGenericLauncher.java:150)
at
com.company.prophet.milliways.shared.common.launcher.MilliwaysGenericLauncher.initService(MilliwaysGenericLauncher.java:120)
at com.company.prophet.projectname.sync.main.Sync.main(Sync.java:108)
Locked ownable synchronizers:
- <0x00000006da73f540> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da73f690> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da84add8> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da84aee0> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da8825d8> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da8c1198> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da912440> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da95eac0> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da95eb98> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daa93360> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daa93470> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dab3b070> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dab3b0c0> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daba1990> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daba19e0> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dabdb580> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dabdb628> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dac3df68> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dac3e010> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dad35b68> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dadb0e88> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dadb0f30> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dadb1018> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dae2e7c8> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dae2e808> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006dae73b20> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daeb57b8> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daeb5870> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daeb5928> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daeb59e0> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daeb5a98> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006daeb5b50> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
client.txt
<http://apache-ignite-users.70518.x6.nabble.com/file/t2757/client.txt>
server1.txt
<http://apache-ignite-users.70518.x6.nabble.com/file/t2757/server1.txt>
server2.txt
<http://apache-ignite-users.70518.x6.nabble.com/file/t2757/server2.txt>
server3.txt
<http://apache-ignite-users.70518.x6.nabble.com/file/t2757/server3.txt>
Any pointers will be useful.
( Attached the thread dump of client and the 3 servers )
regards,
Veena.
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
It is possible that Linux is more lenient towards creation of massive
number of threads.
Regards,
--
Ilya Kasnacheev
ср, 13 янв. 2021 г. в 19:44, VeenaMithare <v....@cmcmarkets.com>:
> Let me debug this a bit more. Also I will try and capture another set of
> thread dumps when this occurs again. This issue occurs on my windows
> machine
> sometimes - doesnt happen all the time. I have not seen this on my linux
> env. yet.
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Client stuck on startup
Posted by VeenaMithare <v....@cmcmarkets.com>.
Let me debug this a bit more. Also I will try and capture another set of
thread dumps when this occurs again. This issue occurs on my windows machine
sometimes - doesnt happen all the time. I have not seen this on my linux
env. yet.
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by VeenaMithare <v....@cmcmarkets.com>.
I dont think the server dumps were taken after the client disconnected . They
were taken when the client was in the stuck state.
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
But are they collected after the client node has already left? Not to
mention that server nodes too have an obscene number of threads, and as
such are vulnerable to the same problem.
Regards,
--
Ilya Kasnacheev
ср, 13 янв. 2021 г. в 18:25, VeenaMithare <v....@cmcmarkets.com>:
> Hello,
>
> Thread dump from the server nodes has been provided in the original post,
>
> regards,
> Veena
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Client stuck on startup
Posted by VeenaMithare <v....@cmcmarkets.com>.
Hello,
Thread dump from the server nodes has been provided in the original post,
regards,
Veena
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
Please provide full thread dump from server nodes.
Of course, you will need to kill client JVM first.
Regards,
--
Ilya Kasnacheev
ср, 13 янв. 2021 г. в 17:09, VeenaMithare <v....@cmcmarkets.com>:
>
> a. If you see the thread dump, it shows these locked synchronizers
> Locked ownable synchronizers:
> - <0x00000006da73f540> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da73f690> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da84add8> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da84aee0> (a
>
>
> If the shutdown is clean, we dont see these locked synchronizers .
>
> Looks like it has registered some state with the server before the shutdown
> hook was invoked. Also this registered state is now preventing further
> restarts .
>
> b. Even though the log says 'invoking shutdown hook' , the jvm does not
> shutdown - it is blocked on starting the ignite.
>
> I will try and see if I can debug more.
>
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Client stuck on startup
Posted by VeenaMithare <v....@cmcmarkets.com>.
a. If you see the thread dump, it shows these locked synchronizers
Locked ownable synchronizers:
- <0x00000006da73f540> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da73f690> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da84add8> (a
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
- <0x00000006da84aee0> (a
If the shutdown is clean, we dont see these locked synchronizers .
Looks like it has registered some state with the server before the shutdown
hook was invoked. Also this registered state is now preventing further
restarts .
b. Even though the log says 'invoking shutdown hook' , the jvm does not
shutdown - it is blocked on starting the ignite.
I will try and see if I can debug more.
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
I'm not sure what happens on the server nodes. Restarting the JVM with this
client should be enough.
Regards,
--
Ilya Kasnacheev
вт, 12 янв. 2021 г. в 19:14, VeenaMithare <v....@cmcmarkets.com>:
> Okay, thanks Ilya . After it gets this issue, the app doesnt startup till
> I
> restart my server nodes.
>
> Is there anyway I can ensure clean shutdown when I face issues like this ?
> It looks like some ignite state is not cleaned up .
>
> regards,
> Veena.
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Client stuck on startup
Posted by VeenaMithare <v....@cmcmarkets.com>.
Okay, thanks Ilya . After it gets this issue, the app doesnt startup till I
restart my server nodes.
Is there anyway I can ensure clean shutdown when I face issues like this ?
It looks like some ignite state is not cleaned up .
regards,
Veena.
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
Shutdown hook means that JVM decided to stop. Probably due to thread
creation error if you ask me.
It seems that the node has never finished initialization, judging by that
log.
Regards,
--
Ilya Kasnacheev
пн, 11 янв. 2021 г. в 19:25, VeenaMithare <v....@cmcmarkets.com>:
> >>Try to use some realistic value such as 64.
>
> I think it hadnt worked earlier - I will try again. client-log.txt
> <http://apache-ignite-users.70518.x6.nabble.com/file/t2757/client-log.txt>
>
>
> Please find the client log file attached. I am seeing 'invoking shutdown
> hook..' as an INFO in the startup logs.
> Is that expected ?
>
> regards,
> Veena.
>
>
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Client stuck on startup
Posted by VeenaMithare <v....@cmcmarkets.com>.
>>Try to use some realistic value such as 64.
I think it hadnt worked earlier - I will try again. client-log.txt
<http://apache-ignite-users.70518.x6.nabble.com/file/t2757/client-log.txt>
Please find the client log file attached. I am seeing 'invoking shutdown
hook..' as an INFO in the startup logs.
Is that expected ?
regards,
Veena.
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
I don't know since it's impossible to check thread dump with such a high
number of threads. I suspect it's possible. I also don't see any log for
client, only thread dump. Can you attach it as well?
Try to use some realistic value such as 64.
Regards,
--
Ilya Kasnacheev
пн, 11 янв. 2021 г. в 18:44, VeenaMithare <v....@cmcmarkets.com>:
> This client is a sync which syncs around 1 million records from our old db
> to
> ignite . When I start the application, it runs the initial sync and I need
> a
> higher striped pool. I think the client sees starvation in striped thread
> pool when I give a small thread pool. ( I dont have the logs for the lower
> thread pool now. ).
>
> Is the striped pool size linked to client startup issue ?
>
>
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>
Re: Client stuck on startup
Posted by VeenaMithare <v....@cmcmarkets.com>.
This client is a sync which syncs around 1 million records from our old db to
ignite . When I start the application, it runs the initial sync and I need a
higher striped pool. I think the client sees starvation in striped thread
pool when I give a small thread pool. ( I dont have the logs for the lower
thread pool now. ).
Is the striped pool size linked to client startup issue ?
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Client stuck on startup
Posted by Ilya Kasnacheev <il...@gmail.com>.
Hello!
Are you sure that you really need a 1024 thread striped pool? What happens
if it's only 16?
Regards,
--
Ilya Kasnacheev
пн, 11 янв. 2021 г. в 17:55, VeenaMithare <v....@cmcmarkets.com>:
> HI ,
>
> One of the clients on my windows box sometimes gets stuck up on startup .
> I
> see this in the thread dump of the client.
>
> a. Not sure what those locked synchronizers are
>
> "main" #1 prio=5 os_prio=0 tid=0x0000000002c9c800 nid=0x4404 waiting on
> condition [0x0000000002c8e000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x00000006fda12d20> (a
> java.util.concurrent.CountDownLatch$Sync)
> at
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
> at
>
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
> at
> java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
> at
>
> org.apache.ignite.spi.discovery.tcp.ClientImpl.spiStart(ClientImpl.java:312)
> at
>
> org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi.spiStart(TcpDiscoverySpi.java:2099)
> at
>
> org.apache.ignite.internal.managers.GridManagerAdapter.startSpi(GridManagerAdapter.java:299)
> at
>
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:943)
> at
>
> org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1960)
> at
> org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:1276)
> at
>
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:2045)
> at
>
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1703)
> - locked <0x00000006db5ca060> (a
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance)
> at
> org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1117)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:637)
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:563)
> at org.apache.ignite.Ignition.start(Ignition.java:321)
> at
>
> com.company.prophet.projectname.dao.projectnameClientUtil.startIgnite(projectnameClientUtil.java:168)
> at
>
> com.company.prophet.projectname.dao.projectnameClientUtil.startIgniteForSingleOrBlueGreen(projectnameClientUtil.java:109)
> at
>
> com.company.prophet.projectname.dao.projectnameClientUtil.startIgnite(projectnameClientUtil.java:84)
> at
>
> com.company.prophet.projectname.dao.projectnameClient.start(projectnameClient.java:126)
> at
> com.company.prophet.projectname.sync.main.Sync.startIgnite(Sync.java:264)
> at
> com.company.prophet.projectname.sync.main.Sync.start(Sync.java:133)
> at
>
> com.company.prophet.milliways.shared.common.launcher.MilliwaysGenericLauncher.run(MilliwaysGenericLauncher.java:69)
> at
>
> com.company.prophet.milliways.shared.common.launcher.MilliwaysGenericLauncher$1.run(MilliwaysGenericLauncher.java:150)
> at
>
> com.company.prophet.milliways.shared.common.launcher.MilliwaysGenericLauncher.initService(MilliwaysGenericLauncher.java:120)
> at
> com.company.prophet.projectname.sync.main.Sync.main(Sync.java:108)
>
> Locked ownable synchronizers:
> - <0x00000006da73f540> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da73f690> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da84add8> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da84aee0> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da8825d8> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da8c1198> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da912440> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da95eac0> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006da95eb98> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daa93360> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daa93470> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dab3b070> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dab3b0c0> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daba1990> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daba19e0> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dabdb580> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dabdb628> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dac3df68> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dac3e010> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dad35b68> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dadb0e88> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dadb0f30> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dadb1018> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dae2e7c8> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dae2e808> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006dae73b20> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daeb57b8> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daeb5870> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daeb5928> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daeb59e0> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daeb5a98> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
> - <0x00000006daeb5b50> (a
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
>
>
> client.txt
> <http://apache-ignite-users.70518.x6.nabble.com/file/t2757/client.txt>
> server1.txt
> <http://apache-ignite-users.70518.x6.nabble.com/file/t2757/server1.txt>
> server2.txt
> <http://apache-ignite-users.70518.x6.nabble.com/file/t2757/server2.txt>
> server3.txt
> <http://apache-ignite-users.70518.x6.nabble.com/file/t2757/server3.txt>
>
> Any pointers will be useful.
>
> ( Attached the thread dump of client and the 3 servers )
>
> regards,
> Veena.
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>