You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@geode.apache.org by "Anilkumar Gingade (Jira)" <ji...@apache.org> on 2020/08/18 19:59:00 UTC
[jira] [Created] (GEODE-8440) Newly restarted node hang in creating
region
Anilkumar Gingade created GEODE-8440:
----------------------------------------
Summary: Newly restarted node hang in creating region
Key: GEODE-8440
URL: https://issues.apache.org/jira/browse/GEODE-8440
Project: Geode
Issue Type: Bug
Components: regions, wan
Reporter: Anilkumar Gingade
In one of the internal test we observed test hanging with:
"vm_9_thr_19_bridge5_host1_31792" #943 daemon prio=5 os_prio=0 cpu=1036.73ms elapsed=1026.10s tid=0x00007fc30c03f000 nid=0x5cbb waiting on condition [0x00007fc2d1ba2000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method)
- parking to wait for <0x00000000e2394520> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.8/LockSupport.java:234)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(java.base@11.0.8/AbstractQueuedSynchronizer.java:1079)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(java.base@11.0.8/AbstractQueuedSynchronizer.java:1369)
at java.util.concurrent.CountDownLatch.await(java.base@11.0.8/CountDownLatch.java:278)
at org.apache.geode.internal.util.concurrent.StoppableCountDownLatch.await(StoppableCountDownLatch.java:72)
at org.apache.geode.distributed.internal.ReplyProcessor21.basicWait(ReplyProcessor21.java:723)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:794)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:771)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:857)
at org.apache.geode.internal.cache.persistence.MembershipFlushRequest.send(MembershipFlushRequest.java:63)
at org.apache.geode.internal.cache.PRHARedundancyProvider.recoverPersistentBuckets(PRHARedundancyProvider.java:1598)
at org.apache.geode.internal.cache.PartitionedRegion.initPRInternals(PartitionedRegion.java:1110)
- locked <0x00000000ec1a8218> (a org.apache.geode.internal.cache.PartitionedRegion)
at org.apache.geode.internal.cache.PartitionedRegion.initialize(PartitionedRegion.java:1220)
at org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:3081)
at org.apache.geode.cache.lucene.internal.LuceneIndexImpl.createRegion(LuceneIndexImpl.java:274)
at org.apache.geode.cache.lucene.internal.LuceneIndexForPartitionedRegion.createRegion(LuceneIndexForPartitionedRegion.java:186)
at org.apache.geode.cache.lucene.internal.LuceneIndexForPartitionedRegion.createLuceneListenersAndFileChunkRegions(LuceneIndexForPartitionedRegion.java:123)
at org.apache.geode.cache.lucene.internal.LuceneIndexImpl.initialize(LuceneIndexImpl.java:150)
at org.apache.geode.cache.lucene.internal.LuceneServiceImpl.afterDataRegionCreated(LuceneServiceImpl.java:372)
at org.apache.geode.cache.lucene.internal.LuceneRegionListener.afterCreate(LuceneRegionListener.java:116)
at org.apache.geode.internal.cache.GemFireCacheImpl.invokeRegionAfter(GemFireCacheImpl.java:3146)
at org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:3123)
at org.apache.geode.internal.cache.GemFireCacheImpl.basicCreateRegion(GemFireCacheImpl.java:2975)
at org.apache.geode.internal.cache.xmlcache.RegionCreation.createRoot(RegionCreation.java:237)
at org.apache.geode.internal.cache.xmlcache.CacheCreation.initializeRegions(CacheCreation.java:675)
at org.apache.geode.internal.cache.xmlcache.CacheCreation.create(CacheCreation.java:594)
at org.apache.geode.internal.cache.xmlcache.CacheXmlParser.create(CacheXmlParser.java:339)
at org.apache.geode.internal.cache.GemFireCacheImpl.loadCacheXml(GemFireCacheImpl.java:4180)
at org.apache.geode.internal.cache.GemFireCacheImpl.initializeDeclarativeCache(GemFireCacheImpl.java:1601)
at org.apache.geode.internal.cache.GemFireCacheImpl.initialize(GemFireCacheImpl.java:1427)
at org.apache.geode.internal.cache.InternalCacheBuilder.create(InternalCacheBuilder.java:191)
- locked <0x00000000e12f5db8> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
- locked <0x00000000e12f5be8> (a java.lang.Class for org.apache.geode.internal.cache.InternalCacheBuilder)
at org.apache.geode.internal.cache.CacheFactoryStatics.create(CacheFactoryStatics.java:61)
at org.apache.geode.cache.CacheFactory.create(CacheFactory.java:352)
at hydra.CacheHelper.createCacheFromXml(CacheHelper.java:215)
- locked <0x00000000e1302218> (a java.lang.Class for hydra.CacheHelper)
at util.CacheUtil.createRegion(CacheUtil.java:376)
at parReg.ParRegTest.HA_reinitializeRegion(ParRegTest.java:806)
at parReg.ParRegTest.HAController(ParRegTest.java:2050)
at parReg.ParRegTest.HydraTask_HAController(ParRegTest.java:1077)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.8/Native Method)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.8/NativeMethodAccessorImpl.java:62)
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.8/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.8/Method.java:566)
at hydra.MethExecutor.execute(MethExecutor.java:173)
at hydra.MethExecutor.execute(MethExecutor.java:141)
at hydra.TestTask.execute(TestTask.java:197)
at hydra.RemoteTestModule$1.run(RemoteTestModule.java:213)
*** Stack changed
"vm_9_thr_19_bridge5_host1_31792" #943 daemon prio=5 os_prio=0 cpu=1036.89ms elapsed=1036.12s tid=0x00007fc30c03f000 nid=0x5cbb waiting on condition [0x00007fc2d1ba2000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method)
- parking to wait for <0x00000000e2394520> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.8/LockSupport.java:234)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(java.base@11.0.8/AbstractQueuedSynchronizer.java:1079)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(java.base@11.0.8/AbstractQueuedSynchronizer.java:1369)
at java.util.concurrent.CountDownLatch.await(java.base@11.0.8/CountDownLatch.java:278)
at org.apache.geode.internal.util.concurrent.StoppableCountDownLatch.await(StoppableCountDownLatch.java:72)
at org.apache.geode.distributed.internal.ReplyProcessor21.basicWait(ReplyProcessor21.java:723)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:794)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:771)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:857)
at org.apache.geode.internal.cache.persistence.MembershipFlushRequest.send(MembershipFlushRequest.java:63)
at org.apache.geode.internal.cache.PRHARedundancyProvider.recoverPersistentBuckets(PRHARedundancyProvider.java:1598)
at org.apache.geode.internal.cache.PartitionedRegion.initPRInternals(PartitionedRegion.java:1110)
- locked <0x00000000ec1a8218> (a org.apache.geode.internal.cache.PartitionedRegion)
at org.apache.geode.internal.cache.PartitionedRegion.initialize(PartitionedRegion.java:1220)
at org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:3081)
at org.apache.geode.cache.lucene.internal.LuceneIndexImpl.createRegion(LuceneIndexImpl.java:274)
at org.apache.geode.cache.lucene.internal.LuceneIndexForPartitionedRegion.createRegion(LuceneIndexForPartitionedRegion.java:186)
at org.apache.geode.cache.lucene.internal.LuceneIndexForPartitionedRegion.createLuceneListenersAndFileChunkRegions(LuceneIndexForPartitionedRegion.java:123)
at org.apache.geode.cache.lucene.internal.LuceneIndexImpl.initialize(LuceneIndexImpl.java:150)
at org.apache.geode.cache.lucene.internal.LuceneServiceImpl.afterDataRegionCreated(LuceneServiceImpl.java:372)
at org.apache.geode.cache.lucene.internal.LuceneRegionListener.afterCreate(LuceneRegionListener.java:116)
at org.apache.geode.internal.cache.GemFireCacheImpl.invokeRegionAfter(GemFireCacheImpl.java:3146)
at org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:3123)
at org.apache.geode.internal.cache.GemFireCacheImpl.basicCreateRegion(GemFireCacheImpl.java:2975)
at org.apache.geode.internal.cache.xmlcache.RegionCreation.createRoot(RegionCreation.java:237)
at org.apache.geode.internal.cache.xmlcache.CacheCreation.initializeRegions(CacheCreation.java:675)
at org.apache.geode.internal.cache.xmlcache.CacheCreation.create(CacheCreation.java:594)
at org.apache.geode.internal.cache.xmlcache.CacheXmlParser.create(CacheXmlParser.java:339)
at org.apache.geode.internal.cache.GemFireCacheImpl.loadCacheXml(GemFireCacheImpl.java:4180)
at org.apache.geode.internal.cache.GemFireCacheImpl.initializeDeclarativeCache(GemFireCacheImpl.java:1601)
at org.apache.geode.internal.cache.GemFireCacheImpl.initialize(GemFireCacheImpl.java:1427)
at org.apache.geode.internal.cache.InternalCacheBuilder.create(InternalCacheBuilder.java:191)
- locked <0x00000000e12f5db8> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
- locked <0x00000000e12f5be8> (a java.lang.Class for org.apache.geode.internal.cache.InternalCacheBuilder)
at org.apache.geode.internal.cache.CacheFactoryStatics.create(CacheFactoryStatics.java:61)
at org.apache.geode.cache.CacheFactory.create(CacheFactory.java:352)
at hydra.CacheHelper.createCacheFromXml(CacheHelper.java:215)
- locked <0x00000000e1302218> (a java.lang.Class for hydra.CacheHelper)
at util.CacheUtil.createRegion(CacheUtil.java:376)
at parReg.ParRegTest.HA_reinitializeRegion(ParRegTest.java:806)
at parReg.ParRegTest.HAController(ParRegTest.java:2050)
at parReg.ParRegTest.HydraTask_HAController(ParRegTest.java:1077)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.8/Native Method)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.8/NativeMethodAccessorImpl.java:62)
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.8/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.8/Method.java:566)
at hydra.MethExecutor.execute(MethExecutor.java:173)
at hydra.MethExecutor.execute(MethExecutor.java:141)
at hydra.TestTask.execute(TestTask.java:197)
at hydra.RemoteTestModule$1.run(RemoteTestModule.java:213)
*** Stack changed
"vm_9_thr_19_bridge5_host1_31792" #943 daemon prio=5 os_prio=0 cpu=1037.46ms elapsed=1075.21s tid=0x00007fc30c03f000 nid=0x5cbb waiting on condition [0x00007fc2d1ba2000]
java.lang.Thread.State: TIMED_WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@11.0.8/Native Method)
- parking to wait for <0x00000000e2394520> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.8/LockSupport.java:234)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(java.base@11.0.8/AbstractQueuedSynchronizer.java:1079)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(java.base@11.0.8/AbstractQueuedSynchronizer.java:1369)
at java.util.concurrent.CountDownLatch.await(java.base@11.0.8/CountDownLatch.java:278)
at org.apache.geode.internal.util.concurrent.StoppableCountDownLatch.await(StoppableCountDownLatch.java:72)
at org.apache.geode.distributed.internal.ReplyProcessor21.basicWait(ReplyProcessor21.java:723)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:794)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:771)
at org.apache.geode.distributed.internal.ReplyProcessor21.waitForRepliesUninterruptibly(ReplyProcessor21.java:857)
at org.apache.geode.internal.cache.persistence.MembershipFlushRequest.send(MembershipFlushRequest.java:63)
at org.apache.geode.internal.cache.PRHARedundancyProvider.recoverPersistentBuckets(PRHARedundancyProvider.java:1598)
at org.apache.geode.internal.cache.PartitionedRegion.initPRInternals(PartitionedRegion.java:1110)
- locked <0x00000000ec1a8218> (a org.apache.geode.internal.cache.PartitionedRegion)
at org.apache.geode.internal.cache.PartitionedRegion.initialize(PartitionedRegion.java:1220)
at org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:3081)
at org.apache.geode.cache.lucene.internal.LuceneIndexImpl.createRegion(LuceneIndexImpl.java:274)
at org.apache.geode.cache.lucene.internal.LuceneIndexForPartitionedRegion.createRegion(LuceneIndexForPartitionedRegion.java:186)
at org.apache.geode.cache.lucene.internal.LuceneIndexForPartitionedRegion.createLuceneListenersAndFileChunkRegions(LuceneIndexForPartitionedRegion.java:123)
at org.apache.geode.cache.lucene.internal.LuceneIndexImpl.initialize(LuceneIndexImpl.java:150)
at org.apache.geode.cache.lucene.internal.LuceneServiceImpl.afterDataRegionCreated(LuceneServiceImpl.java:372)
at org.apache.geode.cache.lucene.internal.LuceneRegionListener.afterCreate(LuceneRegionListener.java:116)
at org.apache.geode.internal.cache.GemFireCacheImpl.invokeRegionAfter(GemFireCacheImpl.java:3146)
at org.apache.geode.internal.cache.GemFireCacheImpl.createVMRegion(GemFireCacheImpl.java:3123)
at org.apache.geode.internal.cache.GemFireCacheImpl.basicCreateRegion(GemFireCacheImpl.java:2975)
at org.apache.geode.internal.cache.xmlcache.RegionCreation.createRoot(RegionCreation.java:237)
at org.apache.geode.internal.cache.xmlcache.CacheCreation.initializeRegions(CacheCreation.java:675)
at org.apache.geode.internal.cache.xmlcache.CacheCreation.create(CacheCreation.java:594)
at org.apache.geode.internal.cache.xmlcache.CacheXmlParser.create(CacheXmlParser.java:339)
at org.apache.geode.internal.cache.GemFireCacheImpl.loadCacheXml(GemFireCacheImpl.java:4180)
at org.apache.geode.internal.cache.GemFireCacheImpl.initializeDeclarativeCache(GemFireCacheImpl.java:1601)
at org.apache.geode.internal.cache.GemFireCacheImpl.initialize(GemFireCacheImpl.java:1427)
at org.apache.geode.internal.cache.InternalCacheBuilder.create(InternalCacheBuilder.java:191)
- locked <0x00000000e12f5db8> (a java.lang.Class for org.apache.geode.internal.cache.GemFireCacheImpl)
- locked <0x00000000e12f5be8> (a java.lang.Class for org.apache.geode.internal.cache.InternalCacheBuilder)
at org.apache.geode.internal.cache.CacheFactoryStatics.create(CacheFactoryStatics.java:61)
at org.apache.geode.cache.CacheFactory.create(CacheFactory.java:352)
at hydra.CacheHelper.createCacheFromXml(CacheHelper.java:215)
- locked <0x00000000e1302218> (a java.lang.Class for hydra.CacheHelper)
at util.CacheUtil.createRegion(CacheUtil.java:376)
at parReg.ParRegTest.HA_reinitializeRegion(ParRegTest.java:806)
at parReg.ParRegTest.HAController(ParRegTest.java:2050)
at parReg.ParRegTest.HydraTask_HAController(ParRegTest.java:1077)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.8/Native Method)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.8/NativeMethodAccessorImpl.java:62)
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.8/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.8/Method.java:566)
at hydra.MethExecutor.execute(MethExecutor.java:173)
at hydra.MethExecutor.execute(MethExecutor.java:141)
at hydra.TestTask.execute(TestTask.java:197)
at hydra.RemoteTestModule$1.run(RemoteTestModule.java:213)
Stack for hung thread vm_9_thr_19_bridge5_host1_31792 was found 5 times and changed 4 times.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)