You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@ignite.apache.org by "Denis Chudov (Jira)" <ji...@apache.org> on 2020/09/09 07:51:00 UTC
[jira] [Created] (IGNITE-13418) Deadlock on multiple cache delete
Denis Chudov created IGNITE-13418:
-------------------------------------
Summary: Deadlock on multiple cache delete
Key: IGNITE-13418
URL: https://issues.apache.org/jira/browse/IGNITE-13418
Project: Ignite
Issue Type: Bug
Reporter: Denis Chudov
Assignee: Denis Chudov
Thread 1:
- acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone
- acquires GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop
- enters H2TreeIndex.destroy
- releases checkpoint read lock in H2Tree.temporaryReleaseLock, can't take it again because of db-checkpoint-thread
Thread 2:
- acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone
- trying to acquire GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop which is held by thread 1
db-checkpoint-thread:
- trying to acquire checkpoint write lock, can't do it because of Thread 2
Decision: H2Tree.temporaryReleaseLock should release lock only in case when tree deletion is asynchronous (H2TreeIndex.destroy is called with async=true), i.e. it happens inside of DurableBackgroundTask. Such tasks are executed in separate threads, which don't hold any other locks.
Thread dump:
{code:java}
Thread [name="sys-#1220%DPL_GRID%DplGridNodeName%", id=3200, state=BLOCKED, blockCnt=1, waitCnt=0]
Lock [object=java.lang.Object@6a9a92ba, ownerName=sys-#1215%DPL_GRID%DplGridNodeName%, ownerId=3195]
at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1695)
at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902)
at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169)
at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644)
at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803)
at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source)
at o.a.i.i.util.IgniteUtils.lambda$null$1(IgniteUtils.java:10879)
at o.a.i.i.util.IgniteUtils$$Lambda$436/321848940.call(Unknown Source)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Locked synchronizers:
java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2582f93c
java.util.concurrent.ThreadPoolExecutor$Worker@4da1cafe
Thread [name="sys-#1215%DPL_GRID%DplGridNodeName%", id=3195, state=BLOCKED, blockCnt=4, waitCnt=437520]
Lock [object=o.a.i.i.processors.failure.FailureProcessor@78edb1e9, ownerName=async-durable-background-task-executor-1-#1222%DPL_GRID%DplGridNodeName%, ownerId=3202]
at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:162)
at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:151)
at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.failCheckpointReadLock(GridCacheDatabaseSharedManager.java:1787)
at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.checkpointReadLock(GridCacheDatabaseSharedManager.java:1722)
at o.a.i.i.processors.query.h2.database.H2Tree.temporaryReleaseLock(H2Tree.java:690)
at o.a.i.i.processors.cache.persistence.tree.BPlusTree.temporaryReleaseLock(BPlusTree.java:2367)
at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2548)
at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2441)
at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2392)
at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy0(H2TreeIndex.java:671)
at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy(H2TreeIndex.java:639)
at o.a.i.i.processors.query.h2.opt.GridH2Table.destroy(GridH2Table.java:567)
at o.a.i.i.processors.query.h2.H2TableDescriptor.onDrop(H2TableDescriptor.java:347)
at o.a.i.i.processors.query.h2.H2Schema.drop(H2Schema.java:127)
at o.a.i.i.processors.query.h2.IgniteH2Indexing.unregisterCache(IgniteH2Indexing.java:2595)
at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1727)
- locked java.lang.Object@6a9a92ba
at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902)
at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169)
at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644)
at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803)
at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source)
at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10898)
at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10800)
at o.a.i.i.processors.cache.GridCacheProcessor.processCacheStopRequestOnExchangeDone(GridCacheProcessor.java:2775)
at o.a.i.i.processors.cache.GridCacheProcessor.onExchangeDone(GridCacheProcessor.java:2899)
at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2210)
at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processFullMessage(GridDhtPartitionsExchangeFuture.java:4197)
at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$1600(GridDhtPartitionsExchangeFuture.java:143)
at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3879)
at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3867)
at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:385)
at o.a.i.i.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:355)
at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveFullMessage(GridDhtPartitionsExchangeFuture.java:3867)
at o.a.i.i.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1798)
at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:408)
at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:395)
at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3492)
at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3471)
at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1106)
at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:589)
at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:386)
at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:312)
at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:102)
at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:301)
at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1574)
at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1198)
at o.a.i.i.managers.communication.GridIoManager.access$4300(GridIoManager.java:132)
at o.a.i.i.managers.communication.GridIoManager$8.run(GridIoManager.java:1099)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748){code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)