You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cloudberry.apache.org by ma...@apache.org on 2025/02/24 05:53:13 UTC

(cloudberry) 01/07: Fix a statement leak involving self-deadlocks

This is an automated email from the ASF dual-hosted git repository.

maxyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry.git

commit d2f1ba727d4cd8da41fed5ad0718bf3f331a4e18
Author: Soumyadeep Chakraborty <so...@gmail.com>
AuthorDate: Tue Feb 6 18:39:47 2024 -0800

    Fix a statement leak involving self-deadlocks
    
    It is possible to leak a slot in a resource queue in the face of
    concurrent self-deadlock, query cancellation/deadlock and an ALTER
    QUEUE operation that bumps the resource queue limit. This can be
    demonstrated with a retail build (asserts-build trips various
    assertions):
    
    1. Session 1: Has tripped the self-deadlock check while trying to lock
    say portal C2, and is currently ERRORing out (it has already been
    granted the extra grant that follows a positive self-deadlock check). We
    haven't reached cleanup yet.
    
    2. Session 2: Is waiting on the same resource queue as Session 1. We
    then induce a query cancel and let's say Session 2 is in the middle of
    ERRORing out and is at the very top of ResLockWaitCancel().
    
    3. Session 3: Bumps the queue limit with ALTER such that portal C2 from
    Session 1 will now fit into the queue.
    
    4. Session 2: Resumes in ResLockWaitCancel() and encounters Session1/C2
    by looking at lock->waitProcs and proc->waitPortalId in:
    ResLockWaitCancel()->ResRemoveFromWaitQueue()->ResCleanupLock()->ResProcLockRemoveSelfAndWakeup()
    It then does an extra grant and limit update for C2, before completing.
    
    5. Session 1: Now proceeds and cleans up the extra grant for C2 from
    Step 1 and errors out.
    
    6. Running gp_resqueue_status shows that there is still 1 active
    statement. This is the leak.
    
    Observation:
    The extra grant and limit update in Step 4 is never cleaned up, which
    causes the leak. Note that this extra grant and limit update could also
    have been caused by a regular deadlock check (instead of query cancellation)
    
    The main reason why we have the leak is other backends can find the
    self-deadlocking process/portal in LOCK->waitProcs and
    PGPROC->waitPortalId.
    
    There is no real need to start waiting before conducting the
    self-deadlock check. Thus, we now do the self-deadlock check much
    earlier.
    
    Reviewed-by: Ashwin Agrawal <aa...@vmware.com>
    Reviewed-by: Xuejing Zhao <zx...@vmware.com>
    Reviewed-by: Jimmy Yih <jy...@vmware.com>
    Reviewed-by: Huansong Fu <fu...@gmail.com>
---
 src/backend/storage/lmgr/proc.c                    | 12 ----
 src/backend/utils/resscheduler/resqueue.c          | 25 +++++--
 .../expected/resource_queue_self_deadlock.out      | 78 ++++++++++++++++++++++
 src/test/isolation2/isolation2_schedule            |  1 +
 .../sql/resource_queue_self_deadlock.sql           | 50 ++++++++++++++
 5 files changed, 147 insertions(+), 19 deletions(-)

diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index f912491825..d70c1be756 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -2256,8 +2256,6 @@ ResProcSleep(LOCKMODE lockmode, LOCALLOCK *locallock, void *incrementSet)
 	uint32		hashcode = locallock->hashcode;
 	LWLockId	partitionLock = LockHashPartitionLock(hashcode);
 
-	bool		selflock = true;		/* initialize result for error. */
-
 	/*
 	 * Don't check my held locks, as we just add at the end of the queue.
 	 */
@@ -2276,16 +2274,6 @@ ResProcSleep(LOCKMODE lockmode, LOCALLOCK *locallock, void *incrementSet)
 
 	MyProc->waitStatus = PROC_WAIT_STATUS_WAITING;	/* initialize result for error */
 
-	/* Now check the status of the self lock footgun. */
-	selflock = ResCheckSelfDeadLock(lock, proclock, incrementSet);
-	if (selflock)
-	{
-		LWLockRelease(partitionLock);
-		ereport(ERROR,
-				(errcode(ERRCODE_T_R_DEADLOCK_DETECTED),
-				 errmsg("deadlock detected, locking against self")));
-	}
-
 	/* Mark that we are waiting for a lock */
 	lockAwaited = locallock;
 
diff --git a/src/backend/utils/resscheduler/resqueue.c b/src/backend/utils/resscheduler/resqueue.c
index c3e95333c6..e7c096e302 100644
--- a/src/backend/utils/resscheduler/resqueue.c
+++ b/src/backend/utils/resscheduler/resqueue.c
@@ -518,6 +518,22 @@ ResLockAcquire(LOCKTAG *locktag, ResPortalIncrement *incrementSet)
 		Assert(status == LIMIT_CHECK_FOUND);
 
 		resLockAcquireStatus = RQA_WAIT_ON_LOCK;
+		/*
+		 * First check if there would be any self-deadlock, before we start
+		 * waiting on the lock.
+		 */
+		if (ResCheckSelfDeadLock(lock, proclock, incrementSet))
+		{
+			LWLockRelease(ResQueueLock);
+			LWLockRelease(partitionLock);
+
+			SIMPLE_FAULT_INJECTOR("res_lock_acquire_self_deadlock_error");
+
+			ereport(ERROR,
+					(errcode(ERRCODE_T_R_DEADLOCK_DETECTED),
+						errmsg("deadlock detected, locking against self")));
+		}
+
 		/*
 		 * The requested lock will exhaust the limit for this resource queue,
 		 * so must wait.
@@ -1247,8 +1263,6 @@ ResWaitOnLock(LOCALLOCK *locallock, ResourceOwner owner, ResPortalIncrement *inc
 
 	/*
 	 * Now sleep.
-	 *
-	 * NOTE: self-deadlocks will throw (do a non-local return).
 	 */
 	if (ResProcSleep(ExclusiveLock, locallock, incrementSet) != LIMIT_CHECK_OK)
 	{
@@ -1476,6 +1490,8 @@ ResRemoveFromWaitQueue(PGPROC *proc, uint32 hashcode)
  * increments. If this exceeds any of the thresholds for the queue then
  * we need to signal that a self deadlock is about to occurr - modulo some
  * footwork for overcommit-able queues.
+ *
+ * Note: ResQueueLock must already be held in Exclusive mode.
  */
 bool
 ResCheckSelfDeadLock(LOCK *lock, PROCLOCK *proclock, ResPortalIncrement *incrementSet)
@@ -1490,9 +1506,6 @@ ResCheckSelfDeadLock(LOCK *lock, PROCLOCK *proclock, ResPortalIncrement *increme
 	bool		memoryThesholdOvercommitted = false;
 	bool		result = false;
 
-	/* Get the resource queue lock before checking the increments. */
-	LWLockAcquire(ResQueueLock, LW_EXCLUSIVE);
-
 	/* Get the queue for this lock. */
 	queue = GetResQueueFromLock(lock);
 	limits = queue->limits;
@@ -1578,8 +1591,6 @@ ResCheckSelfDeadLock(LOCK *lock, PROCLOCK *proclock, ResPortalIncrement *increme
 		/* our caller will throw an ERROR. */
 	}
 
-	LWLockRelease(ResQueueLock);
-
 	return result;
 }
 
diff --git a/src/test/isolation2/expected/resource_queue_self_deadlock.out b/src/test/isolation2/expected/resource_queue_self_deadlock.out
new file mode 100644
index 0000000000..473092d3f6
--- /dev/null
+++ b/src/test/isolation2/expected/resource_queue_self_deadlock.out
@@ -0,0 +1,78 @@
+-- This test is used to test if waiting on a resource queue lock will trigger a
+-- self deadlock detection and if we will properly clean up (preventing nasty
+-- race conditions)
+
+0: CREATE RESOURCE QUEUE rq_self WITH (active_statements = 1);
+CREATE
+0: CREATE ROLE r_self RESOURCE QUEUE rq_self;
+CREATE
+
+1: SET ROLE r_self;
+SET
+1: BEGIN;
+BEGIN
+1: DECLARE c1 CURSOR FOR SELECT 1;
+DECLARE
+
+0: SELECT gp_inject_fault_infinite('res_lock_acquire_self_deadlock_error', 'suspend', dbid) FROM gp_segment_configuration WHERE content = -1 AND role = 'p';
+ gp_inject_fault_infinite 
+--------------------------
+ Success:                 
+(1 row)
+0&: SELECT gp_wait_until_triggered_fault('res_lock_acquire_self_deadlock_error', 1, dbid) FROM gp_segment_configuration WHERE content = -1 AND role = 'p';  <waiting ...>
+-- Will trip a self-deadlock and the backend will be suspended.
+1&: DECLARE c2 CURSOR FOR SELECT 1;  <waiting ...>
+
+2: SET ROLE r_self;
+SET
+-- Will wait for a slot in the queue.
+2&: SELECT 33176;  <waiting ...>
+
+-- Now inflate the active statement count for the queue.
+0<:  <... completed>
+ gp_wait_until_triggered_fault 
+-------------------------------
+ Success:                      
+(1 row)
+0: ALTER RESOURCE QUEUE rq_self WITH (active_statements = 5);
+ALTER
+
+-- Cancellation of the backend should not lead to us doing an extra grant to the
+-- process undergoing self-deadlock. Doing so, leads to an assertion failure:
+-- FailedAssertion("!(lock->nGranted <= lock->nRequested)", File: "resqueue.c", Line: 1086)
+-- ResGrantLock resqueue.c:1086
+-- ResProcLockRemoveSelfAndWakeup resqueue.c:1361
+-- ResCleanUpLock resqueue.c:1215
+-- ResRemoveFromWaitQueue resqueue.c:1473
+-- ResLockWaitCancel proc.c:2102
+-- ResLockPortal resscheduler.c:695
+-- ...
+3: SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE query='SELECT 33176;';
+ pg_cancel_backend 
+-------------------
+ t                 
+(1 row)
+2<:  <... completed>
+ERROR:  canceling statement due to user request
+
+0: SELECT gp_inject_fault('res_lock_acquire_self_deadlock_error', 'reset', dbid) FROM gp_segment_configuration WHERE content = -1 AND role = 'p';
+ gp_inject_fault 
+-----------------
+ Success:        
+(1 row)
+
+1<:  <... completed>
+ERROR:  deadlock detected, locking against self
+
+-- Sanity check: Ensure that the resource queue is now empty.
+0: SELECT rsqcountlimit, rsqcountvalue from pg_resqueue_status WHERE rsqname = 'rq_self';
+ rsqcountlimit | rsqcountvalue 
+---------------+---------------
+ 5             | 0             
+(1 row)
+
+-- Clean up the test
+0: DROP ROLE r_self;
+DROP
+0: DROP RESOURCE QUEUE rq_self;
+DROP
diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule
index 9d887b4d43..5eb38be485 100644
--- a/src/test/isolation2/isolation2_schedule
+++ b/src/test/isolation2/isolation2_schedule
@@ -17,6 +17,7 @@ test: select_dropped_table
 test: update_hash_col_utilitymode execute_on_utilitymode
 
 # Tests on global deadlock detector
+test: resource_queue_self_deadlock
 test: gdd/prepare
 test: gdd/concurrent_update
 test: gdd/dist-deadlock-01 gdd/dist-deadlock-04 gdd/dist-deadlock-05 gdd/dist-deadlock-06 gdd/dist-deadlock-07 gdd/dist-deadlock-102 gdd/dist-deadlock-103 gdd/dist-deadlock-104 gdd/dist-deadlock-106 gdd/non-lock-105
diff --git a/src/test/isolation2/sql/resource_queue_self_deadlock.sql b/src/test/isolation2/sql/resource_queue_self_deadlock.sql
new file mode 100644
index 0000000000..ab0c5d9704
--- /dev/null
+++ b/src/test/isolation2/sql/resource_queue_self_deadlock.sql
@@ -0,0 +1,50 @@
+-- This test is used to test if waiting on a resource queue lock will trigger a
+-- self deadlock detection and if we will properly clean up (preventing nasty
+-- race conditions)
+
+0: CREATE RESOURCE QUEUE rq_self WITH (active_statements = 1);
+0: CREATE ROLE r_self RESOURCE QUEUE rq_self;
+
+1: SET ROLE r_self;
+1: BEGIN;
+1: DECLARE c1 CURSOR FOR SELECT 1;
+
+0: SELECT gp_inject_fault_infinite('res_lock_acquire_self_deadlock_error', 'suspend', dbid) FROM
+    gp_segment_configuration WHERE content = -1 AND role = 'p';
+0&: SELECT gp_wait_until_triggered_fault('res_lock_acquire_self_deadlock_error', 1, dbid)
+    FROM gp_segment_configuration WHERE content = -1 AND role = 'p';
+-- Will trip a self-deadlock and the backend will be suspended.
+1&: DECLARE c2 CURSOR FOR SELECT 1;
+
+2: SET ROLE r_self;
+-- Will wait for a slot in the queue.
+2&: SELECT 33176;
+
+-- Now inflate the active statement count for the queue.
+0<:
+0: ALTER RESOURCE QUEUE rq_self WITH (active_statements = 5);
+
+-- Cancellation of the backend should not lead to us doing an extra grant to the
+-- process undergoing self-deadlock. Doing so, leads to an assertion failure:
+-- FailedAssertion("!(lock->nGranted <= lock->nRequested)", File: "resqueue.c", Line: 1086)
+-- ResGrantLock resqueue.c:1086
+-- ResProcLockRemoveSelfAndWakeup resqueue.c:1361
+-- ResCleanUpLock resqueue.c:1215
+-- ResRemoveFromWaitQueue resqueue.c:1473
+-- ResLockWaitCancel proc.c:2102
+-- ResLockPortal resscheduler.c:695
+-- ...
+3: SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE query='SELECT 33176;';
+2<:
+
+0: SELECT gp_inject_fault('res_lock_acquire_self_deadlock_error', 'reset', dbid) FROM
+    gp_segment_configuration WHERE content = -1 AND role = 'p';
+
+1<:
+
+-- Sanity check: Ensure that the resource queue is now empty.
+0: SELECT rsqcountlimit, rsqcountvalue from pg_resqueue_status WHERE rsqname = 'rq_self';
+
+-- Clean up the test
+0: DROP ROLE r_self;
+0: DROP RESOURCE QUEUE rq_self;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cloudberry.apache.org
For additional commands, e-mail: commits-help@cloudberry.apache.org