You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/05/07 05:37:04 UTC

(spark) branch master updated: [SPARK-51821][CORE] Call interrupt() without holding uninterruptibleLock to avoid possible deadlock

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new bb0b2d29d37e [SPARK-51821][CORE] Call interrupt() without holding uninterruptibleLock to avoid possible deadlock
bb0b2d29d37e is described below

commit bb0b2d29d37e393280253fb4cb95fee461efaace
Author: Vlad Rozov <vr...@amazon.com>
AuthorDate: Wed May 7 13:36:44 2025 +0800

    [SPARK-51821][CORE] Call interrupt() without holding uninterruptibleLock to avoid possible deadlock
    
    ### What changes were proposed in this pull request?
    Do not hold `uninterruptibleLock` monitor while calling `super.interrupt()` in `UninterruptibleThread`, instead use newly introduced `awaitInterruptThread` flag and wait for `super.interrupt()` to be called.
    
    ### Why are the changes needed?
    There is potential deadlock as `UninterruptibleThread` may be blocked on NIO operation and interrupting channel while holding `uninterruptibleLock` monitor may cause deadlock like in
    ```
    Found one Java-level deadlock:
    =============================
    "pool-1-thread-1-ScalaTest-running-UninterruptibleThreadSuite":
      waiting to lock monitor 0x00006000036ee3c0 (object 0x000000070f3019d0, a java.lang.Object),
      which is held by "task thread"
    
    "task thread":
      waiting to lock monitor 0x00006000036e75a0 (object 0x000000070f70fe80, a java.lang.Object),
      which is held by "pool-1-thread-1-ScalaTest-running-UninterruptibleThreadSuite"
    
    Java stack information for the threads listed above:
    ===================================================
    "pool-1-thread-1-ScalaTest-running-UninterruptibleThreadSuite":
            at java.nio.channels.spi.AbstractInterruptibleChannel$1.interrupt(java.base17.0.14/AbstractInterruptibleChannel.java:157)
            - waiting to lock <0x000000070f3019d0> (a java.lang.Object)
            at java.lang.Thread.interrupt(java.base17.0.14/Thread.java:1004)
            - locked <0x000000070f70fc90> (a java.lang.Object)
            at org.apache.spark.util.UninterruptibleThread.interrupt(UninterruptibleThread.scala:99)
            - locked <0x000000070f70fe80> (a java.lang.Object)
            at org.apache.spark.util.UninterruptibleThreadSuite.$anonfun$new$5(UninterruptibleThreadSuite.scala:159)
            - locked <0x000000070f70f9f8> (a java.lang.Object)
            at org.apache.spark.util.UninterruptibleThreadSuite$$Lambda$216/0x000000700120d6c8.apply$mcV$sp(Unknown Source)
            at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
            at org.scalatest.enablers.Timed$$anon$1.timeoutAfter(Timed.scala:127)
            at org.scalatest.concurrent.TimeLimits$.failAfterImpl(TimeLimits.scala:282)
            at org.scalatest.concurrent.TimeLimits.failAfter(TimeLimits.scala:231)
            at org.scalatest.concurrent.TimeLimits.failAfter$(TimeLimits.scala:230)
            at org.apache.spark.SparkFunSuite.failAfter(SparkFunSuite.scala:69)
            at org.apache.spark.SparkFunSuite.$anonfun$test$2(SparkFunSuite.scala:155)
            at org.apache.spark.SparkFunSuite$$Lambda$205/0x0000007001207700.apply(Unknown Source)
            at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
            at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
            at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
            at org.scalatest.Transformer.apply(Transformer.scala:22)
            at org.scalatest.Transformer.apply(Transformer.scala:20)
            at org.scalatest.funsuite.AnyFunSuiteLike$$anon$1.apply(AnyFunSuiteLike.scala:226)
            at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:227)
            at org.scalatest.funsuite.AnyFunSuiteLike.invokeWithFixture$1(AnyFunSuiteLike.scala:224)
            at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTest$1(AnyFunSuiteLike.scala:236)
            at org.scalatest.funsuite.AnyFunSuiteLike$$Lambda$343/0x00000070012867b0.apply(Unknown Source)
            at org.scalatest.SuperEngine.runTestImpl(Engine.scala:306)
            at org.scalatest.funsuite.AnyFunSuiteLike.runTest(AnyFunSuiteLike.scala:236)
            at org.scalatest.funsuite.AnyFunSuiteLike.runTest$(AnyFunSuiteLike.scala:218)
            at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:69)
            at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:234)
            at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:227)
            at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:69)
            at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTests$1(AnyFunSuiteLike.scala:269)
            at org.scalatest.funsuite.AnyFunSuiteLike$$Lambda$339/0x00000070012833e0.apply(Unknown Source)
            at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:413)
            at org.scalatest.SuperEngine$$Lambda$340/0x0000007001283998.apply(Unknown Source)
            at scala.collection.immutable.List.foreach(List.scala:334)
            at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
            at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:396)
            at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:475)
            at org.scalatest.funsuite.AnyFunSuiteLike.runTests(AnyFunSuiteLike.scala:269)
            at org.scalatest.funsuite.AnyFunSuiteLike.runTests$(AnyFunSuiteLike.scala:268)
            at org.scalatest.funsuite.AnyFunSuite.runTests(AnyFunSuite.scala:1564)
            at org.scalatest.Suite.run(Suite.scala:1114)
            at org.scalatest.Suite.run$(Suite.scala:1096)
            at org.scalatest.funsuite.AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run(AnyFunSuite.scala:1564)
            at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$run$1(AnyFunSuiteLike.scala:273)
            at org.scalatest.funsuite.AnyFunSuiteLike$$Lambda$332/0x000000700127b000.apply(Unknown Source)
            at org.scalatest.SuperEngine.runImpl(Engine.scala:535)
            at org.scalatest.funsuite.AnyFunSuiteLike.run(AnyFunSuiteLike.scala:273)
            at org.scalatest.funsuite.AnyFunSuiteLike.run$(AnyFunSuiteLike.scala:272)
            at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:69)
            at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
            at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
            at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
            at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:69)
            at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:321)
            at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:517)
            at sbt.ForkMain$Run.lambda$runTest$1(ForkMain.java:414)
            at sbt.ForkMain$Run$$Lambda$107/0x0000007001110000.call(Unknown Source)
            at java.util.concurrent.FutureTask.run(java.base17.0.14/FutureTask.java:264)
            at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base17.0.14/ThreadPoolExecutor.java:1136)
            at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base17.0.14/ThreadPoolExecutor.java:635)
            at java.lang.Thread.run(java.base17.0.14/Thread.java:840)
    "task thread":
            at org.apache.spark.util.UninterruptibleThread.interrupt(UninterruptibleThread.scala:96)
            - waiting to lock <0x000000070f70fe80> (a java.lang.Object)
            at org.apache.spark.util.UninterruptibleThreadSuite$InterruptibleChannel.implCloseChannel(UninterruptibleThreadSuite.scala:143)
            at java.nio.channels.spi.AbstractInterruptibleChannel.close(java.base17.0.14/AbstractInterruptibleChannel.java:112)
            - locked <0x000000070f3019d0> (a java.lang.Object)
            at org.apache.spark.util.UninterruptibleThreadSuite$InterruptibleChannel.<init>(UninterruptibleThreadSuite.scala:138)
            at org.apache.spark.util.UninterruptibleThreadSuite$$anon$5.run(UninterruptibleThreadSuite.scala:153)
    
    Found 1 deadlock.
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Added 2 new test cases to the `UninterruptibleThreadSuite`
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No
    
    Closes #50594 from vrozov/uninterruptible.
    
    Authored-by: Vlad Rozov <vr...@amazon.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../apache/spark/util/UninterruptibleThread.scala  | 125 ++++++++++++++++-----
 .../spark/util/UninterruptibleThreadSuite.scala    |  57 +++++++++-
 2 files changed, 149 insertions(+), 33 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala b/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
index 24788d69121b..8fba5ed944c6 100644
--- a/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
+++ b/core/src/main/scala/org/apache/spark/util/UninterruptibleThread.scala
@@ -35,21 +35,90 @@ private[spark] class UninterruptibleThread(
     this(null, name)
   }
 
-  /** A monitor to protect "uninterruptible" and "interrupted" */
-  private val uninterruptibleLock = new Object
+  private class UninterruptibleLock {
+    /**
+     * Indicates if `this`  thread are in the uninterruptible status. If so, interrupting
+     * "this" will be deferred until `this`  enters into the interruptible status.
+     */
+    @GuardedBy("uninterruptibleLock")
+    private var uninterruptible = false
 
-  /**
-   * Indicates if `this`  thread are in the uninterruptible status. If so, interrupting
-   * "this" will be deferred until `this`  enters into the interruptible status.
-   */
-  @GuardedBy("uninterruptibleLock")
-  private var uninterruptible = false
+    /**
+     * Indicates if we should interrupt `this` when we are leaving the uninterruptible zone.
+     */
+    @GuardedBy("uninterruptibleLock")
+    private var shouldInterruptThread = false
 
-  /**
-   * Indicates if we should interrupt `this` when we are leaving the uninterruptible zone.
-   */
-  @GuardedBy("uninterruptibleLock")
-  private var shouldInterruptThread = false
+    /**
+     * Indicates that we should wait for interrupt() call before proceeding.
+     */
+    @GuardedBy("uninterruptibleLock")
+    private var awaitInterruptThread = false
+
+    /**
+     * Set [[uninterruptible]] to given value and returns the previous value.
+     */
+    def getAndSetUninterruptible(value: Boolean): Boolean = synchronized {
+      val uninterruptible = this.uninterruptible
+      this.uninterruptible = value
+      uninterruptible
+    }
+
+    def setShouldInterruptThread(value: Boolean): Unit = synchronized {
+      shouldInterruptThread = value
+    }
+
+    def setAwaitInterruptThread(value: Boolean): Unit = synchronized {
+      awaitInterruptThread = value
+    }
+
+    /**
+     * Is call to [[java.lang.Thread.interrupt()]] pending
+     */
+    def isInterruptPending: Boolean = synchronized {
+      // Clear the interrupted status if it's set.
+      shouldInterruptThread = Thread.interrupted() || shouldInterruptThread
+      // wait for super.interrupt() to be called
+      !shouldInterruptThread && awaitInterruptThread
+    }
+
+    /**
+     * Set [[uninterruptible]] back to false and call [[java.lang.Thread.interrupt()]] to
+     * recover interrupt state if necessary
+     */
+    def recoverInterrupt(): Unit = synchronized {
+      uninterruptible = false
+      if (shouldInterruptThread) {
+        shouldInterruptThread = false
+        // Recover the interrupted status
+        UninterruptibleThread.super.interrupt()
+      }
+    }
+
+    /**
+     * Is it safe to call [[java.lang.Thread.interrupt()]] and interrupt the current thread
+     * @return true when there is no concurrent [[runUninterruptibly()]] call ([[uninterruptible]]
+     *         is true) and no concurrent [[interrupt()]] call, otherwise false
+     */
+    def isInterruptible: Boolean = synchronized {
+      shouldInterruptThread = uninterruptible
+      // as we are releasing uninterruptibleLock before calling super.interrupt() there is a
+      // possibility that runUninterruptibly() would be called after lock is released but before
+      // super.interrupt() is called. In this case to prevent runUninterruptibly() from being
+      // interrupted, we use awaitInterruptThread flag. We need to set it only if
+      // runUninterruptibly() is not yet set uninterruptible to true (!shouldInterruptThread) and
+      // there is no other threads that called interrupt (awaitInterruptThread is already true)
+      if (!shouldInterruptThread && !awaitInterruptThread) {
+        awaitInterruptThread = true
+        true
+      } else {
+        false
+      }
+    }
+  }
+
+  /** A monitor to protect "uninterruptible" and "interrupted" */
+  private val uninterruptibleLock = new UninterruptibleLock
 
   /**
    * Run `f` uninterruptibly in `this` thread. The thread won't be interrupted before returning
@@ -63,27 +132,23 @@ private[spark] class UninterruptibleThread(
         s"Expected: $this but was ${Thread.currentThread()}")
     }
 
-    if (uninterruptibleLock.synchronized { uninterruptible }) {
+    if (uninterruptibleLock.getAndSetUninterruptible(true)) {
       // We are already in the uninterruptible status. So just run "f" and return
       return f
     }
 
-    uninterruptibleLock.synchronized {
-      // Clear the interrupted status if it's set.
-      shouldInterruptThread = Thread.interrupted() || shouldInterruptThread
-      uninterruptible = true
+    while (uninterruptibleLock.isInterruptPending) {
+      try {
+        Thread.sleep(100)
+      } catch {
+        case _: InterruptedException => uninterruptibleLock.setShouldInterruptThread(true)
+      }
     }
+
     try {
       f
     } finally {
-      uninterruptibleLock.synchronized {
-        uninterruptible = false
-        if (shouldInterruptThread) {
-          // Recover the interrupted status
-          super.interrupt()
-          shouldInterruptThread = false
-        }
-      }
+      uninterruptibleLock.recoverInterrupt()
     }
   }
 
@@ -92,11 +157,11 @@ private[spark] class UninterruptibleThread(
    * interrupted until it enters into the interruptible status.
    */
   override def interrupt(): Unit = {
-    uninterruptibleLock.synchronized {
-      if (uninterruptible) {
-        shouldInterruptThread = true
-      } else {
+    if (uninterruptibleLock.isInterruptible) {
+      try {
         super.interrupt()
+      } finally {
+        uninterruptibleLock.setAwaitInterruptThread(false)
       }
     }
   }
diff --git a/core/src/test/scala/org/apache/spark/util/UninterruptibleThreadSuite.scala b/core/src/test/scala/org/apache/spark/util/UninterruptibleThreadSuite.scala
index 9c0ee1e1303e..fbc954d05af8 100644
--- a/core/src/test/scala/org/apache/spark/util/UninterruptibleThreadSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UninterruptibleThreadSuite.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.util
 
+import java.nio.channels.spi.AbstractInterruptibleChannel
 import java.util.concurrent.{CountDownLatch, TimeUnit}
 
 import scala.util.Random
@@ -115,6 +116,45 @@ class UninterruptibleThreadSuite extends SparkFunSuite {
     assert(interruptStatusBeforeExit)
   }
 
+  test("no runUninterruptibly") {
+    @volatile var hasInterruptedException = false
+    val t = new UninterruptibleThread("test") {
+      override def run(): Unit = {
+        if (sleep(0)) {
+          hasInterruptedException = true
+        }
+      }
+    }
+    t.interrupt()
+    t.start()
+    t.join()
+    assert(hasInterruptedException === true)
+  }
+
+  test("SPARK-51821 uninterruptibleLock deadlock") {
+    val latch = new CountDownLatch(1)
+    val task = new UninterruptibleThread("task thread") {
+      override def run(): Unit = {
+        val channel = new AbstractInterruptibleChannel() {
+          override def implCloseChannel(): Unit = {
+            begin()
+            latch.countDown()
+            try {
+              Thread.sleep(Long.MaxValue)
+            } catch {
+              case _: InterruptedException => Thread.currentThread().interrupt()
+            }
+          }
+        }
+        channel.close()
+      }
+    }
+    task.start()
+    assert(latch.await(10, TimeUnit.SECONDS), "await timeout")
+    task.interrupt()
+    task.join()
+  }
+
   test("stress test") {
     @volatile var hasInterruptedException = false
     val t = new UninterruptibleThread("test") {
@@ -148,9 +188,20 @@ class UninterruptibleThreadSuite extends SparkFunSuite {
       }
     }
     t.start()
-    for (i <- 0 until 400) {
-      Thread.sleep(Random.nextInt(10))
-      t.interrupt()
+    val threads = new Array[Thread](10)
+    for (j <- 0 until 10) {
+      threads(j) = new Thread() {
+        override def run(): Unit = {
+          for (i <- 0 until 400) {
+            Thread.sleep(Random.nextInt(10))
+            t.interrupt()
+          }
+        }
+      }
+      threads(j).start()
+    }
+    for (j <- 0 until 10) {
+      threads(j).join()
     }
     t.join()
     assert(hasInterruptedException === false)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org