You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ignite.apache.org by "Ignite TC Bot (Jira)" <ji...@apache.org> on 2020/09/09 18:59:00 UTC
[jira] [Commented] (IGNITE-13418) Deadlock on multiple cache delete
[ https://issues.apache.org/jira/browse/IGNITE-13418?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17193131#comment-17193131 ]
Ignite TC Bot commented on IGNITE-13418:
----------------------------------------
{panel:title=Branch: [pull/8229/head] Base: [master] : No blockers found!|borderStyle=dashed|borderColor=#ccc|titleBGColor=#D6F7C1}{panel}
{panel:title=Branch: [pull/8229/head] Base: [master] : New Tests (2)|borderStyle=dashed|borderColor=#ccc|titleBGColor=#D6F7C1}
{color:#00008b}PDS (Indexing){color} [[tests 2|https://ci.ignite.apache.org/viewLog.html?buildId=5593099]]
* {color:#013220}IgnitePdsWithIndexingTestSuite: MultipleParallelCacheDeleteDeadlockTest.test - PASSED{color}
* {color:#013220}IgnitePdsWithIndexingTestSuite: LongDestroyDurableBackgroundTaskTest.testRemoveIndexesOnTableDrop - PASSED{color}
{panel}
[TeamCity *--> Run :: All* Results|https://ci.ignite.apache.org/viewLog.html?buildId=5593140&buildTypeId=IgniteTests24Java8_RunAll]
> Deadlock on multiple cache delete
> ---------------------------------
>
> Key: IGNITE-13418
> URL: https://issues.apache.org/jira/browse/IGNITE-13418
> Project: Ignite
> Issue Type: Bug
> Reporter: Denis Chudov
> Assignee: Denis Chudov
> Priority: Major
> Time Spent: 10m
> Remaining Estimate: 0h
>
> Thread 1:
> - acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone
> - acquires GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop
> - enters H2TreeIndex.destroy
> - releases checkpoint read lock in H2Tree.temporaryReleaseLock, can't take it again because of db-checkpoint-thread
> Thread 2:
> - acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone
> - trying to acquire GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop which is held by thread 1
> db-checkpoint-thread:
> - trying to acquire checkpoint write lock, can't do it because of Thread 2
> Decision: H2Tree.temporaryReleaseLock should release lock only in case when tree deletion is asynchronous (H2TreeIndex.destroy is called with async=true), i.e. it happens inside of DurableBackgroundTask. Such tasks are executed in separate threads, which don't hold any other locks.
> Thread dump:
> {code:java}
> Thread [name="sys-#1220%DPL_GRID%DplGridNodeName%", id=3200, state=BLOCKED, blockCnt=1, waitCnt=0]
> Lock [object=java.lang.Object@6a9a92ba, ownerName=sys-#1215%DPL_GRID%DplGridNodeName%, ownerId=3195]
> at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1695)
> at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902)
> at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169)
> at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644)
> at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803)
> at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source)
> at o.a.i.i.util.IgniteUtils.lambda$null$1(IgniteUtils.java:10879)
> at o.a.i.i.util.IgniteUtils$$Lambda$436/321848940.call(Unknown Source)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Locked synchronizers:
> java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2582f93c
> java.util.concurrent.ThreadPoolExecutor$Worker@4da1cafe
> Thread [name="sys-#1215%DPL_GRID%DplGridNodeName%", id=3195, state=BLOCKED, blockCnt=4, waitCnt=437520]
> Lock [object=o.a.i.i.processors.failure.FailureProcessor@78edb1e9, ownerName=async-durable-background-task-executor-1-#1222%DPL_GRID%DplGridNodeName%, ownerId=3202]
> at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:162)
> at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:151)
> at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.failCheckpointReadLock(GridCacheDatabaseSharedManager.java:1787)
> at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.checkpointReadLock(GridCacheDatabaseSharedManager.java:1722)
> at o.a.i.i.processors.query.h2.database.H2Tree.temporaryReleaseLock(H2Tree.java:690)
> at o.a.i.i.processors.cache.persistence.tree.BPlusTree.temporaryReleaseLock(BPlusTree.java:2367)
> at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2548)
> at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
> at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
> at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
> at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2441)
> at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2392)
> at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy0(H2TreeIndex.java:671)
> at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy(H2TreeIndex.java:639)
> at o.a.i.i.processors.query.h2.opt.GridH2Table.destroy(GridH2Table.java:567)
> at o.a.i.i.processors.query.h2.H2TableDescriptor.onDrop(H2TableDescriptor.java:347)
> at o.a.i.i.processors.query.h2.H2Schema.drop(H2Schema.java:127)
> at o.a.i.i.processors.query.h2.IgniteH2Indexing.unregisterCache(IgniteH2Indexing.java:2595)
> at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1727)
> - locked java.lang.Object@6a9a92ba
> at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902)
> at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169)
> at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644)
> at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803)
> at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source)
> at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10898)
> at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10800)
> at o.a.i.i.processors.cache.GridCacheProcessor.processCacheStopRequestOnExchangeDone(GridCacheProcessor.java:2775)
> at o.a.i.i.processors.cache.GridCacheProcessor.onExchangeDone(GridCacheProcessor.java:2899)
> at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2210)
> at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processFullMessage(GridDhtPartitionsExchangeFuture.java:4197)
> at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$1600(GridDhtPartitionsExchangeFuture.java:143)
> at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3879)
> at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3867)
> at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:385)
> at o.a.i.i.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:355)
> at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveFullMessage(GridDhtPartitionsExchangeFuture.java:3867)
> at o.a.i.i.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1798)
> at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:408)
> at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:395)
> at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3492)
> at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3471)
> at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1106)
> at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:589)
> at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:386)
> at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:312)
> at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:102)
> at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:301)
> at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1574)
> at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1198)
> at o.a.i.i.managers.communication.GridIoManager.access$4300(GridIoManager.java:132)
> at o.a.i.i.managers.communication.GridIoManager$8.run(GridIoManager.java:1099)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748){code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)